From: Peter Palfrader Date: Sat, 14 Sep 2019 13:20:59 +0000 (+0200) Subject: move things from modules/roles/static* to modules/static* X-Git-Url: https://git.adam-barratt.org.uk/?a=commitdiff_plain;h=0c067639ec7d7050b69249b85e6aa83ab91820b5;p=mirror%2Fdsa-puppet.git move things from modules/roles/static* to modules/static* Files and headers etc. have not yet been modified. That's the next step. This was strictly a git mv. --- diff --git a/modules/roles/files/static-mirroring/OVERVIEW b/modules/roles/files/static-mirroring/OVERVIEW deleted file mode 100644 index 18365946e..000000000 --- a/modules/roles/files/static-mirroring/OVERVIEW +++ /dev/null @@ -1,63 +0,0 @@ - -- static-update-component is run by the user on the source host. - If not run under sudo as the staticuser already, it sudos to the staticuser, - re-execing itself. It them sshs to the static-master for that component to - run static-master-update-component. - - LOCKING: - none, but see static-master-update-component - -- static-master-update-component is run on the static-master. - It rsyncs the contents from the source host to the static master, and then - triggers static-master-run to push the content to the mirrors. - - The sync happens to a new -updating.incoming-XXXXXX directory. On - sync success, is replaced with that new tree, and the - static-master-run trigger happens. - - LOCKING: - - exclusive locks are held on - - .lock - -- static-master-run triggers all the mirrors for a component to initiate syncs. - When all mirrors have an up-to-date tree, they are instructed to update - the cur-> symlink to the new tree. - - To begin with, static-master-run copies to -current-push. - This is the tree all the mirrors then sync from. If the push was successful, - -current-push is renamed to -current-live. - - LOCKING: - - exclusive locks are held on - - .lock - -- static-mirror-run runs on a mirror and syncs components. - There is a symlink called 'cur' that points to either tree-a or tree-b for - each component. the cur tree is the one that is live, the other one usually - does not exist, except when a sync is ongoing (or a previous one failed and - we keep a partial tree). - - During a sync, we sync to the tree- that is not the live one. When instructed by - static-master-run, we update the symlink and remove the old tree. - - static-mirror-run rsyncs either -current-push or -current-live for a component. - - LOCKING: - during all of static-mirror-run, we keep an exclusive lock on the - dir, i.e., the directory that holds tree-[ab] and cur. - -- static-mirror-run-all - - Run static-mirror-run for all components on this mirror, fetching the -live- tree. - - LOCKING: - none, but see static-mirror-run. - -- staticsync-ssh-wrap - - wrapper for ssh job dispatching on source, master, and mirror. - - LOCKING: - - on master, when syncing -live- trees: - a shared lock is held on .lock during - the rsync process. diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run deleted file mode 100755 index 95c9355a5..000000000 --- a/modules/roles/files/static-mirroring/static-master-run +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/python - -import fcntl -import os -import shutil -import subprocess -import string -import sys -import tempfile -import time - -serialname = '.serial' -had_warnings = False - -conffile = '/etc/staticsync.conf' -config={} - -with open(conffile) as f: - for line in f: - line = line.rstrip() - if not line or line.startswith("#"): continue - (name, value) = line.split("=") - config[name] = value - -for key in ('base',): - if not key in config: - raise Exception("Configuration element '%s' not found in config file %s"%(key, conffile)) - -allclients = set() -with open('/etc/static-clients.conf') as f: - for line in f: - line = line.strip() - if line == "": continue - if line.startswith('#'): continue - allclients.add(line) - -def log(m): - t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime()) - print t, m - -def stage1(pipes, status, clients): - for c in clients: - p = pipes[c] - while 1: - line = p.stdout.readline() - if line == '': - status[c] = 'failed' - p.stdout.close() - p.stdin.close() - p.wait() - log("%s: failed with returncode %d"%(c,p.returncode)) - break - - line = line.strip() - log("%s >> %s"%(c, line)) - if not line.startswith('[MSM]'): continue - kw = string.split(line, ' ', 2)[1] - - if kw == 'ALREADY-CURRENT': - pipes[c].stdout.close() - pipes[c].stdin.close() - p.wait() - if p.returncode == 0: - log("%s: already current"%(c,)) - status[c] = 'ok' - else: - log("%s: said ALREADY-CURRENT but returncode %d"%(c,p.returncode)) - status[c] = 'failed' - break - elif kw == 'STAGE1-DONE': - log("%s: waiting"%(c,)) - status[c] = 'waiting' - break - elif kw in ['STAGE1-START']: - pass - else: - log("%s: ignoring unknown line"%(c,)) - -def count_statuses(status): - cnt = {} - for k in status: - v = status[k] - if v not in cnt: cnt[v] = 1 - else: cnt[v] += 1 - return cnt - -def stage2(pipes, status, command, clients): - for c in clients: - if status[c] != 'waiting': continue - log("%s << %s"%(c, command)) - pipes[c].stdin.write("%s\n"%(command,)) - - for c in clients: - if status[c] != 'waiting': continue - p = pipes[c] - - (o, dummy) = p.communicate('') - for l in string.split(o, "\n"): - log("%s >> %s"%(c, l)) - log("%s: returned %d"%(c, p.returncode)) - -def callout(component, serial, clients): - log("Calling clients...") - pipes = {} - status = {} - for c in clients: - args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)] - p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - pipes[c] = p - status[c] = 'in-progress' - - log("Stage 1...") - stage1(pipes, status, clients) - log("Stage 1 done.") - cnt = count_statuses(status) - - if 'failed' in cnt and cnt['failed'] >= 2: - log("%d clients failed, aborting..."%(cnt['failed'],)) - stage2(pipes, status, 'abort', clients) - return False - - failedmirrorsfile = os.path.join(config['base'], 'master', component + "-failedmirrors") - if 'failed' in cnt: - log("WARNING: %d clients failed! Continuing anyway!"%(cnt['failed'],)) - global had_warnings - had_warnings = True - f = open(failedmirrorsfile, "w") - for c in status: - if status[c] == 'failed': f.write(c+"\n") - f.close() - else: - if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile) - - if 'waiting' in cnt: - log("Committing...") - stage2(pipes, status, 'go', clients) - return True - else: - log("All clients up to date.") - return True - -def load_component_info(component): - with open('/etc/static-components.conf') as f: - for line in f: - if line.startswith('#'): continue - field = line.strip().split() - if len(field) < 4: continue - if field[1] != component: continue - meta = {} - meta['master'] = field[0] - meta['sourcehost'] = field[2] - meta['sourcedir'] = field[3] - meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set() - meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set() - return meta - else: - return None - -cleanup_dirs = [] -def run_mirror(component): - meta = load_component_info(component) - if meta is None: - log("Component %s not found."%(component,)) - return False - clients = allclients - meta['extraignoreclients'] - - # setup - basemaster = os.path.join(config['base'], 'master') - componentdir = os.path.join(basemaster, component) - cur = componentdir + '-current-push' - live = componentdir + '-current-live' - tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new); - tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old); - os.chmod(tmpdir_new, 0755) - - locks = [] - lockfiles = [ os.path.join(basemaster, component + ".lock") ] - for p in lockfiles: - fd = os.open(p, os.O_RDONLY) - log("Acquiring lock for %s(%d)."%(p,fd)) - fcntl.flock(fd, fcntl.LOCK_EX) - locks.append(fd) - log("All locks acquired.") - - for p in (live, ): - if not os.path.exists(p): os.mkdir(p, 0755) - - #for p in (componentdir, live, tmpdir_new): - # if not os.path.exists(p): os.mkdir(p, 0755) - # fd = os.open(p, os.O_RDONLY) - # log("Acquiring lock for %s(%d)."%(p,fd)) - # fcntl.flock(fd, fcntl.LOCK_EX) - # locks.append(fd) - #log("All locks acquired.") - - serialfile = os.path.join(componentdir, serialname) - try: - with open(serialfile) as f: serial = int(f.read()) - except: - serial = int(time.time()) - with open(serialfile, "w") as f: f.write("%d\n"%(serial,)) - log("Serial is %s."%(serial,)) - - log("Populating %s."%(tmpdir_new,)) - subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new]) - - if os.path.exists(cur): - log("Removing existing %s."%(cur,)) - shutil.rmtree(cur) - - log("Renaming %s to %s."%(tmpdir_new, cur)) - os.rename(tmpdir_new, cur) - - proceed = callout(component, serial, clients) - - if proceed: - log("Moving %s aside."%(live,)) - os.rename(live, os.path.join(tmpdir_old, 'old')) - log("Renaming %s to %s."%(cur, live)) - os.rename(cur, live) - log("Cleaning up.") - shutil.rmtree(tmpdir_old) - if had_warnings: log("Done, with warnings.") - else: log("Done.") - ret = True - else: - log("Aborted.") - ret = False - - for fd in locks: - os.close(fd) - - return ret - - -if len(sys.argv) != 2: - print >> sys.stderr, "Usage: %s "%(sys.argv[0],) - sys.exit(1) -component = sys.argv[1] - -ok = False -try: - ok = run_mirror(component) -finally: - for p in cleanup_dirs: - if os.path.exists(p): shutil.rmtree(p) - -if not ok: - sys.exit(1) -# vim:set et: -# vim:set ts=2: -# vim:set shiftwidth=2: diff --git a/modules/roles/files/static-mirroring/static-master-update-component b/modules/roles/files/static-mirroring/static-master-update-component deleted file mode 100755 index 4ab5143e5..000000000 --- a/modules/roles/files/static-mirroring/static-master-update-component +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash - -# Updates one component (i.e. subdirectory) in static-master/master - -# acquires a shared lock on the base directory (so that we know no updates are -# outgoing, as those acquire an exclusive one). Also acquired an exclusive lock -# on the component directory in question. -# -# The config file is a list of component source-directory pairs. - -# Copyright (c) 2012 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -componentlist=/etc/static-components.conf -. /etc/staticsync.conf -if ! [ -n "$masterbase" ]; then - echo >&2 "masterbase not configured!" - exit 1 -fi - -set -e -set -u - -if [ "`id -u`" != "`stat -c %u "$masterbase"`" ]; then - echo >&2 "You are probably running this as the wrong user." - exit 1 -fi - -lock() { - local fd="$1"; shift - local path="$1"; shift - local exclusive="$1"; shift - - eval "exec $fd< '$path'" - - if [ "$exclusive" -gt 0 ]; then - locktype="-e" - else - locktype="-s" - fi - - if ! flock "$locktype" "$fd"; then - echo >&2 "$0: Cannot acquire lock on $path (flock $locktype failed) - Very bad, we should have waited!" - exit 1 - fi -} - -unlock() { - local fd="$1"; shift - - if ! flock -o "$fd"; then - echo >&2 "$0: Cannot release lock on fd $fd - This should not have happened!" - exit 1 - fi - eval "exec $fd<&-" -} - -if [ "$#" != 1 ]; then - echo >&2 "Usage: $0 " - exit 1 -fi - -component="$1" - -if [ "${component%/*}" != "$component" ] ; then - echo >&2 "$0: Invalid component: $component"; - exit 1 -fi - -srchost="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $3; exit}' "$componentlist")" -srcdir="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $4; exit}' "$componentlist")" -if [ -z "$srchost" ] || [ -z "$srcdir" ]; then - echo >&2 "$0: Invalid component: $component (not found in $componentlist)"; - exit 1 -fi - -tgtlock="$masterbase/$component.lock" -if ! [ -e "$tgtlock" ]; then - touch "$tgtlock" -fi -echo "$0: Acquiring lock on $tgtlock..." -lock 203 "$tgtlock" 1 - -tgt="$masterbase/$component" -if ! [ -d "$tgt" ]; then - echo "$0: Creating $tgt for $component"; - mkdir "$tgt" -fi - -if [ "$srchost" = "`hostname -f`" ]; then - src="$srcdir" -else - src="$srchost:$srcdir" -fi - -#echo "$0: Acquiring lock on $tgt..." -#lock 201 "$tgt" 1 - -tmpdir_new="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.incoming-XXXXXX")" -tmpdir_old="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.removing-XXXXXX")" -trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT -chmod 0755 "$tmpdir_new" - -#echo "$0: Acquiring lock on $tmpdir_new..." -#lock 202 "$tmpdir_new" 1 -echo "$0: Got them." - -echo "$0: Updating master copy of $component..." -rsync --delete \ - -trz \ - --links --hard-links --safe-links \ - --link-dest="$tgt" \ - --exclude='/.serial' \ - "$src/." "$tmpdir_new/." -echo "$0: Done. Committing." - -mv --no-target-directory "$tgt" "$tmpdir_old/old" -if ! mv --no-target-directory "$tmpdir_new" "$tgt"; then - echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt. Trying to recover" - rm -rf "$tgt" - mv --no-target-directory "$tmpdir_old/old" "$tgt" - echo >&2 "$0: Rolled back to old tree, maybe even successfully." - exit 1 -fi - -rm -rf "$tmpdir_new" "$tmpdir_old" -trap - EXIT - -date '+%s' > "$tgt/.serial" -#unlock 201 -#unlock 202 -unlock 203 -echo "$0: Triggering mirror runs..." -exec static-master-run "$component" - -# vim:set et: -# vim:set ts=2: -# vim:set shiftwidth=2: diff --git a/modules/roles/files/static-mirroring/static-mirror-run b/modules/roles/files/static-mirroring/static-mirror-run deleted file mode 100755 index eac646bb3..000000000 --- a/modules/roles/files/static-mirroring/static-mirror-run +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/bash - -# initiate a staged mirror update from sync-source for a component. -# -# if we have a serial file and we got a serial on the command line, only sync if the serial is different - -# Copyright (c) 2012 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -set -e -set -u - -NAME="$(basename "$0")" - -usage() { - echo "Usage: $0 [--one-stage] []" -} - -if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then usage; exit 0; fi - -one_stage=0 -while :; do - case "${1:-}" in - --) - shift - break; - ;; - --one-stage) - shift - one_stage=1 - ;; - -*) usage >&2 - exit 1 - ;; - *) - break - ;; - esac -done - -COMPONENTDIR=${1:-}; shift -SYNC_SOURCE=${1:-}; shift -SYNC_SERIAL=${1:-}; shift || true -if [ -z "$COMPONENTDIR" ]; then usage >&2; exit 1; fi -if [ -z "$SYNC_SOURCE" ]; then usage >&2; exit 1; fi -COMPONENT="$(basename "${COMPONENTDIR}")" - -RSYNC="rsync" -RSYNC_BASE_OPTIONS="-rtvz --delete --links --hard-links --safe-links" -RSYNC_SSH_OPTIONS="ssh -o AddressFamily=inet -o BatchMode=yes" - -LOGDIR="$HOME/logs" -LOGFILE="$LOGDIR/$NAME-run-${COMPONENT}.log" - - -ALPHA="tree-a" -BRAVO="tree-b" -ACTIVE="cur" - -CNF_FILE="$HOME/etc/$NAME.conf" -! [ -e "$CNF_FILE" ] || . "$CNF_FILE" - -SOURCE="${SYNC_SOURCE}/" -COMPONENTDIR="${COMPONENTDIR}/" - -############################################### - -# point stdout and stderr to the logfile if it's not a tty. -# save stdout to fd5 for communications with the master - -log_setup() { - mkdir -p "$LOGDIR" - if ! [ -t 1 ]; then - # move current stdout to fd5 and reopen to logfile - exec 5>&1- - exec 1>> "$LOGFILE" - else - # duplicate stdout to fd5 - exec 5>&1 - fi - if ! [ -t 2 ]; then - exec 2>> "$LOGFILE" - fi -} - -log() { - echo "[$(date)][$NAME][$$] $1" -} - -lock() { - mkdir -p "$COMPONENTDIR" - exec 200< "$COMPONENTDIR" - if ! flock -e 200; then - log "Cannot acquire lock." - echo >&5 "[MSM] LOCK-ERROR" - exit 1 - fi - log "Got the lock." -} - -############################################### - - -log_setup -log "called with $*" -lock - -if [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" = "$ALPHA" ] ; then - staging="$BRAVO" - active="$ALPHA" -elif [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" != "$BRAVO" ] ; then - echo >&5 "Invalid state of ${COMPONENTDIR}${ACTIVE}." - exit 1 -else - staging="$ALPHA" - active="$BRAVO" -fi -log "active is $active; staging is $staging" - -rsync_source="${SOURCE}" -rsync_curactive="${COMPONENTDIR}${active}/" -rsync_target="${COMPONENTDIR}${staging}/" - -if [ -e "$rsync_curactive/.serial" ] && [ -n "$SYNC_SERIAL" ] && [ "$(cat $rsync_curactive/.serial)" = "$SYNC_SERIAL" ]; then - log "active is already at serial $SYNC_SERIAL. No action required." - echo >&5 "[MSM] ALREADY-CURRENT" - exit 0 -fi - -echo >&5 "[MSM] STAGE1-START" -log "Running $RSYNC $RSYNC_BASE_OPTIONS -e $RSYNC_SSH_OPTIONS --link-dest $rsync_curactive $rsync_source $rsync_target" -$RSYNC $RSYNC_BASE_OPTIONS -e "$RSYNC_SSH_OPTIONS" --link-dest "$rsync_curactive" "$rsync_source" "$rsync_target" -log "rsync done." -echo >&5 "[MSM] STAGE1-DONE" -if [ "$one_stage" -gt 0 ]; then - action="go" -else - read action -fi - -case "$action" in - go) - ln --symbolic --force --no-target-directory "$staging" "${COMPONENTDIR}$ACTIVE" - rm -rf "$rsync_curactive" - echo >&5 "[MSM] STAGE2-DONE" - log "stage2 done" - ;; - abort) - echo >&5 "[MSM] STAGE2-ABORT" - log "stage2 abort" - ;; - *) - echo >&5 "[MSM] STAGE2-UNKNOWN-ACTION $action" - log "stage2 unknown action $action" - exit 1 - ;; -esac - -savelog "$LOGFILE" diff --git a/modules/roles/files/static-mirroring/static-mirror-run-all b/modules/roles/files/static-mirroring/static-mirror-run-all deleted file mode 100755 index c6aea0c10..000000000 --- a/modules/roles/files/static-mirroring/static-mirror-run-all +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# initiate a mirror run for all components on this host. - -# Copyright (c) 2012, 2015 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -set -u - -. /etc/staticsync.conf -if ! [ -n "$base" ]; then - echo >&2 "base not configured!" - exit 1 -fi - -awk -v host="$(hostname -f)" ' - !/^ *(#|$)/ { - split($6,ignorehosts,",") - for (i in ignorehosts) { - if (host == ignorehosts[i]) { - next - } - } - print $1, $2 - }' /etc/static-components.conf | - while read master component ; do - static-mirror-run --one-stage "$base/mirrors/$component" "$master:$component/-live-" - done diff --git a/modules/roles/files/static-mirroring/static-update-component b/modules/roles/files/static-mirroring/static-update-component deleted file mode 100755 index 455d17d38..000000000 --- a/modules/roles/files/static-mirroring/static-update-component +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2012 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -usage() { - echo >&2 "Usage: $0 " - exit 1 -} - -componentlist=/etc/static-components.conf -. /etc/staticsync.conf -if ! [ -n "$staticuser" ]; then - echo >&2 "staticuser not configured!" - exit 1 -fi - -if [ "$#" = 1 ]; then - component="$1" -else - usage -fi - - -if [ "${component%/*}" != "$component" ] ; then - echo >&2 "$0: Invalid component: $component"; - exit 1 -fi - -thishost=$(hostname -f) -masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$componentlist")" -srchost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $3; exit}' "$componentlist")" -srcdir="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $4; exit}' "$componentlist")" -inextralist="$( - awk -v component="$component" -v host="$thishost" ' - !/^ *(#|$)/ && $2 == component { - split($5,extra,",") - for (i in extra) { - if (host == extra[i]) { - printf "%s:%s\n", $3, $4 - exit - } - } - exit - }' "$componentlist" - )" -if [ -z "$srchost" ] || [ -z "$srcdir" ]; then - echo >&2 "$0: Invalid component: $component (not found in $componentlist)"; - exit 1 -fi - -if ! [ "$srchost" = "$thishost" ] && [ -z "$inextralist" ]; then - echo >&2 "Component $component is sourced from $srchost, and this host is neither that nor in the extra allowed list." - exit 1 -fi - -if [ "$srchost" = "$thishost" ] && ! [ -d "$srcdir" ]; then - echo >&2 "Component source directory $srcdir does not exist or is not a directory, or is not accessible." - exit 1 -fi - -if [ "`id -nu`" != "$staticuser" ]; then - sudo -u "$staticuser" static-update-component "$@" -else - ssh -o AddressFamily=inet -t -t -o ServerAliveInterval=300 -o PreferredAuthentications=publickey "$masterhost" static-master-update-component "$component" -fi - -# vim:set et: -# vim:set ts=2: -# vim:set shiftwidth=2: -# vim:set syn=sh: diff --git a/modules/roles/files/static-mirroring/staticsync-ssh-wrap b/modules/roles/files/static-mirroring/staticsync-ssh-wrap deleted file mode 100755 index 959b4f8f0..000000000 --- a/modules/roles/files/static-mirroring/staticsync-ssh-wrap +++ /dev/null @@ -1,225 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2009, 2010, 2012 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -set -e -set -u - -. /etc/staticsync.conf -if ! [ -n "$base" ]; then - echo >&2 "base not configured!" - exit 1 -fi -BASEDIR="$base" - -MYLOGNAME="`basename "$0"`[$$]" -COMPONENTLIST=/etc/static-components.conf - -usage() { - echo "local Usage: $0 " - echo "via ssh orig command:" - echo " mirror " - echo " rsync " - echo " static-master-update-component " -} - -one_more_arg() { - if [ "$#" -lt 1 ]; then - usage >&2 - exit 1 - fi -} - -info() { - logger -p daemon.info -t "$MYLOGNAME" "$1" -} - -croak() { - logger -s -p daemon.warn -t "$MYLOGNAME" "$1" - exit 1 -} - -do_mirror() { - local remote_host="$1"; shift - one_more_arg "$@" - local component="$1"; shift - one_more_arg "$@" - local serial="$1"; shift - - masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$COMPONENTLIST")" - if [ -z "$masterhost" ]; then - croak "Did not find master for component $component." - elif [ "$masterhost" != "$remote_host" ]; then - croak "$remote_host is not master for $component." - else - info "Host $remote_host triggered a mirror run for $component, serial $serial" - exec /usr/local/bin/static-mirror-run "$BASEDIR/mirrors/$component" "$remote_host:$component/-new-" "$serial" - echo >&2 "Exec failed" - croak "exec failed" - fi -} - -do_rsync_on_master() { - local remote_host="$1"; shift - local allowed_rsyncs - allowed_rsyncs=() - allowed_rsyncs+=("--server --sender -vlHtrze.iLsf --safe-links .") # wheezy - allowed_rsyncs+=("--server --sender -vlHtrze.iLsfx --safe-links .") # jessie - allowed_rsyncs+=("--server --sender -vlHtrze.iLsfxC --safe-links .") # stretch - - for cmd_idx in ${!allowed_rsyncs[*]}; do - args="${allowed_rsyncs[$cmd_idx]}" - for component in $(awk -v this_host="$(hostname -f)" '!/^ *(#|$)/ && $1 == this_host {print $2}' $COMPONENTLIST); do - if [ "$*" = "$args $component/-new-/" ] || [ "$*" = "$args ./$component/-new-/" ] ; then - local path="$BASEDIR/master/$component-current-push" - info "serving $remote_host with $path" - exec rsync $args "$path/." - croak "Exec failed" - elif [ "$*" = "$args $component/-live-/" ] || [ "$*" = "$args ./$component/-live-/" ] ; then - local path="$BASEDIR/master/$component-current-live" - info "host $remote_host wants $path, acquiring lock" - tgtlock="$BASEDIR/master/$component.lock" - if ! [ -e "$tgtlock" ]; then - touch "$tgtlock" - fi - exec 200< "$tgtlock" - if ! flock -s -w 0 200; then - echo >&2 "Cannot acquire shared lock on $tgtlock covering $path - this should mean an update is already underway anyway." - exit 1 - fi - exec rsync $args "$path/." - croak "Exec failed" - fi - done - done -} - -do_rsync_on_source() { - local remote_host="$1" - shift - - local allowed_rsyncs - allowed_rsyncs=() - - if [ -e "$COMPONENTLIST" ]; then - for path in $(awk -v host="$(hostname -f)" '!/^ *(#|$)/ && $3 == host {print $4}' $COMPONENTLIST); do - allowed_rsyncs+=("--server --sender -lHtrze.iLsf --safe-links . $path/.") # wheezy - allowed_rsyncs+=("--server --sender -lHtrze.iLsfx --safe-links . $path/.") # jessie - allowed_rsyncs+=("--server --sender -lHtrze.iLsfxC --safe-links . $path/.") # stretch - done - fi - for cmd_idx in ${!allowed_rsyncs[*]}; do - allowed="${allowed_rsyncs[$cmd_idx]}" - if [ "$*" = "$allowed" ]; then - info "Running for host $remote_host: rsync $*" - exec rsync "$@" - croak "Exec failed" - fi - done -} - -do_rsync() { - do_rsync_on_master "$@" - do_rsync_on_source "$@" - - info "NOT allowed for $remote_host: rsync $*" - echo >&2 "This rsync command ($@) not allowed." - exit 1 -} - -do_update_component() { - local remote_host="$1"; shift - - one_more_arg "$@" - component="$1" - shift - - hit="$( - awk -v this_host="$(hostname -f)" -v component="$component" -v host="$remote_host" ' - !/^ *(#|$)/ && $1 == this_host && $2 == component { - if ($3 == host) { - print $4 - exit - } - split($5,extra,",") - for (i in extra) { - if (host == extra[i]) { - printf "%s:%s\n", $3, $4 - exit - } - } - exit - }' "$COMPONENTLIST" - )" - if [ -n "$hit" ]; then - exec static-master-update-component "$component" - echo >&2 "Exec failed" - croak "exec failed" - else - info "Not whitelisted: $remote_host update $component" - echo >&2 "Not whitelisted: $remote_host update $component" - exit 1 - fi -} - - -if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 -fi - -one_more_arg "$@" -remote_host="$1" -shift - - -# check/parse remote command line -if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then - croak "Did not find SSH_ORIGINAL_COMMAND" -fi -set "dummy" ${SSH_ORIGINAL_COMMAND} -shift - -info "host $remote_host called with $*" - -one_more_arg "$@" -action="$1" -shift - -case "$action" in - # on a static mirror, update a component from its master - mirror) - do_mirror "$remote_host" "$@" - ;; - # on a static source, allow fetching from the master, - # on a master, allow fetching from a component's mirrors - rsync) - do_rsync "$remote_host" "$@" - ;; - # on a master, initiate an update of a component - static-master-update-component) - do_update_component "$remote_host" "$@" - ;; - *) - croak "Invalid operation '$action'" - ;; -esac diff --git a/modules/roles/manifests/static/base.pp b/modules/roles/manifests/static/base.pp deleted file mode 100644 index 89a0a383c..000000000 --- a/modules/roles/manifests/static/base.pp +++ /dev/null @@ -1,31 +0,0 @@ -# the base class defining things common for all three static classes (master, mirror, source) -class roles::static::base { - $query = 'nodes[certname] { resources { type = "Class" and title = "Roles::Static_mirror" } }' - $static_mirrors = sort(puppetdb_query($query).map |$value| { $value["certname"] }) - - file { '/etc/static-components.conf': - content => template('roles/static-mirroring/static-components.conf.erb'), - } - - file { '/usr/local/bin/staticsync-ssh-wrap': - source => 'puppet:///modules/roles/static-mirroring/staticsync-ssh-wrap', - mode => '0555', - } - - file { '/usr/local/bin/static-update-component': - source => 'puppet:///modules/roles/static-mirroring/static-update-component', - mode => '0555', - } - - file { '/etc/staticsync.conf': - content => @("EOF"), - # This file is sourced by bash - # and parsed by python - # - empty lines and lines starting with a # are ignored. - # - other lines are key=value. No extra spaces anywhere. No quoting. - base=/srv/static.debian.org - masterbase=/home/staticsync/static-master/master - staticuser=staticsync - | EOF - } -} diff --git a/modules/roles/manifests/static/srvdir.pp b/modules/roles/manifests/static/srvdir.pp deleted file mode 100644 index 7fef3830c..000000000 --- a/modules/roles/manifests/static/srvdir.pp +++ /dev/null @@ -1,13 +0,0 @@ -# create the directory on static hosts and disable backups -class roles::static::srvdir { - file { '/srv/static.debian.org': - ensure => directory, - mode => '0755', - owner => 'staticsync', - group => 'staticsync', - } - - file { '/srv/static.debian.org/.nobackup': - content => '', - } -} diff --git a/modules/roles/manifests/static/ssh.pp b/modules/roles/manifests/static/ssh.pp deleted file mode 100644 index 22078cb60..000000000 --- a/modules/roles/manifests/static/ssh.pp +++ /dev/null @@ -1,20 +0,0 @@ -# wrapper for ssh setup for statichosts -class roles::static::ssh( - Variant[Array[String], String] $add_tag, - String $collect_tag, - ) -{ - ssh::keygen {'staticsync': } - - ssh::authorized_key_add { 'staticsync': - target_user => 'staticsync', - command => "/usr/local/bin/staticsync-ssh-wrap ${::fqdn}", - key => $facts['staticsync_key'], - options => ['restrict', 'pty'], - collect_tag => $add_tag, - } - ssh::authorized_key_collect { 'staticsync': - target_user => 'staticsync', - collect_tag => $collect_tag, - } -} diff --git a/modules/roles/manifests/static_master.pp b/modules/roles/manifests/static_master.pp deleted file mode 100644 index 6a36ed3fc..000000000 --- a/modules/roles/manifests/static_master.pp +++ /dev/null @@ -1,38 +0,0 @@ -# static master -# -# each component defines exactly one static master. Content is copied from the source host -# to the master, and from there to all the mirrors. -# -class roles::static_master { - include roles::static::base - include roles::static::srvdir - - # masters need to talk to mirrors and sources and themselves - class { 'roles::static::ssh': - add_tag => [ 'staticsync-mirror', 'staticsync-source', 'staticsync-master' ], - collect_tag => 'staticsync-master', - } - - file { '/usr/local/bin/static-master-run': - source => 'puppet:///modules/roles/static-mirroring/static-master-run', - mode => '0555', - } - file { '/usr/local/bin/static-master-update-component': - source => 'puppet:///modules/roles/static-mirroring/static-master-update-component', - mode => '0555', - } - file { '/etc/static-clients.conf': - content => template('roles/static-mirroring/static-clients.conf.erb'), - } - - file { '/home/staticsync/static-master': - ensure => link, - target => '/srv/static.debian.org', - } - file { '/srv/static.debian.org/master': - ensure => directory, - mode => '0755', - owner => 'staticsync', - group => 'staticsync', - } -} diff --git a/modules/roles/manifests/static_mirror.pp b/modules/roles/manifests/static_mirror.pp deleted file mode 100644 index 380ad3dac..000000000 --- a/modules/roles/manifests/static_mirror.pp +++ /dev/null @@ -1,31 +0,0 @@ -# a static mirror -# -# this receives pushes from the master and then usually serves the content to the public -class roles::static_mirror { - include roles::static::base - include roles::static::srvdir - - # mirrors talk only to masters - class { 'roles::static::ssh': - add_tag => 'staticsync-master', - collect_tag => 'staticsync-mirror', - } - - file { '/usr/local/bin/static-mirror-run': - source => 'puppet:///modules/roles/static-mirroring/static-mirror-run', - mode => '0555', - } - - file { '/usr/local/bin/static-mirror-run-all': - source => 'puppet:///modules/roles/static-mirroring/static-mirror-run-all', - mode => '0555', - } - - file { '/etc/cron.d/puppet-static-mirror': ensure => absent, } - concat::fragment { 'puppet-crontab--static-mirror': - target => '/etc/cron.d/puppet-crontab', - content => @(EOF) - @reboot staticsync sleep 60; chronic static-mirror-run-all - | EOF - } -} diff --git a/modules/roles/manifests/static_source.pp b/modules/roles/manifests/static_source.pp deleted file mode 100644 index 960366cf9..000000000 --- a/modules/roles/manifests/static_source.pp +++ /dev/null @@ -1,19 +0,0 @@ -# a static source -# -# origin of static content. From here it goes to the static master before that one pushes it to the mirrors -class roles::static_source { - include roles::static::base - - if ! defined(Class["roles::static_master"]) { - # sources talk only to masters, but only set this up if we are not - # *also* a static master since we cannot have two meaningful roles::static:ssh - # instances in the current setup. - # - # this adds the limitation that the master of any component whose source is also a - # master node needs to have that same host as its master and not some other one. - class { 'roles::static::ssh': - add_tag => 'staticsync-master', - collect_tag => 'staticsync-source', - } - } -} diff --git a/modules/roles/templates/static-mirroring/static-clients.conf.erb b/modules/roles/templates/static-mirroring/static-clients.conf.erb deleted file mode 100644 index 6d2cd5bfe..000000000 --- a/modules/roles/templates/static-mirroring/static-clients.conf.erb +++ /dev/null @@ -1,15 +0,0 @@ -## -## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. -## - -<%= - -# do not include mirrors in static_mirror_nopush -static_mirror_nopush = scope.lookupvar('deprecated::roles')['static_mirror_nopush'] - -scope.lookupvar('deprecated::roles')['static_mirror'].reject{ |x| static_mirror_nopush.include?(x) }.join("\n") - -# vim:set et: -# vim:set sts=4 ts=4: -# vim:set shiftwidth=4: -%> diff --git a/modules/roles/templates/static-mirroring/static-components.conf.erb b/modules/roles/templates/static-mirroring/static-components.conf.erb deleted file mode 100644 index c04a3a56d..000000000 --- a/modules/roles/templates/static-mirroring/static-components.conf.erb +++ /dev/null @@ -1,66 +0,0 @@ -## -## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. -## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git -## - -<%= - -lines = [] - -lines << "# This file has been autogenerated and pushed by puppet. Edit static-components.yaml in puppet." -lines << "# " - -# this is the list of static mirrors, or, technically, the list of -# nodes with the roles::static_mirror class applied to it. this should -# be populated from outside the template from PuppetDB, see: -# modules/roles/manifests/static_base.pp -mirrors = @static_mirrors - -config = YAML.load(File.open('/etc/puppet/modules/roles/misc/static-components.yaml').read) - - -config['mirrors'].each do |mirror, mc| - if not mirrors.include?(mirror) - fail("static-components.yaml defines mirror #{mirror} but we do not know of it") - end -end - -config['components'].each_pair do |component_name, component_conf| - %w{exclude-mirrors extra-push limit-mirrors}.each do |key| - component_conf[key] = [] unless component_conf.has_key?(key) - end - - srchost, srcpath = component_conf['source'].split(':', 2) - - if not component_conf['exclude-mirrors'].empty? and \ - not component_conf['limit-mirrors'].empty? then - fail("Component #{component_name} specifies both exclude-mirrors and limit-mirrors.") - end - - # In the end, we care about an exclude list, so let's invert limit-mirror and turn it into an exclude list - if not component_conf['limit-mirrors'].empty? then - component_conf['exclude-mirrors'] = mirrors.select do |m| - not component_conf['limit-mirrors'].include?(m) - end - end - - # mirrors may also specify limits as components-include (thus excluding all others). Apply this - config['mirrors'].each do |mirror, mc| - mirror_components_include = mc.fetch('components-include', []) - next if mirror_components_include.empty? - - if not mirror_components_include.include?(component_name) - next if component_conf['exclude-mirrors'].include?(mirror) # do not exclude twice - component_conf['exclude-mirrors'] << mirror - end - end - - exclude = component_conf['exclude-mirrors'].sort().join(',') - exclude = '-' if exclude == "" - extrapush = component_conf['extra-push'].sort().join(',') - extrapush = '-' if extrapush == "" - - lines << "#{component_conf['master']} #{component_name} #{srchost} #{srcpath} #{extrapush} #{exclude}" -end -lines.join("\n") -%> diff --git a/modules/staticsync/files/OVERVIEW b/modules/staticsync/files/OVERVIEW new file mode 100644 index 000000000..18365946e --- /dev/null +++ b/modules/staticsync/files/OVERVIEW @@ -0,0 +1,63 @@ + +- static-update-component is run by the user on the source host. + If not run under sudo as the staticuser already, it sudos to the staticuser, + re-execing itself. It them sshs to the static-master for that component to + run static-master-update-component. + + LOCKING: + none, but see static-master-update-component + +- static-master-update-component is run on the static-master. + It rsyncs the contents from the source host to the static master, and then + triggers static-master-run to push the content to the mirrors. + + The sync happens to a new -updating.incoming-XXXXXX directory. On + sync success, is replaced with that new tree, and the + static-master-run trigger happens. + + LOCKING: + - exclusive locks are held on + - .lock + +- static-master-run triggers all the mirrors for a component to initiate syncs. + When all mirrors have an up-to-date tree, they are instructed to update + the cur-> symlink to the new tree. + + To begin with, static-master-run copies to -current-push. + This is the tree all the mirrors then sync from. If the push was successful, + -current-push is renamed to -current-live. + + LOCKING: + - exclusive locks are held on + - .lock + +- static-mirror-run runs on a mirror and syncs components. + There is a symlink called 'cur' that points to either tree-a or tree-b for + each component. the cur tree is the one that is live, the other one usually + does not exist, except when a sync is ongoing (or a previous one failed and + we keep a partial tree). + + During a sync, we sync to the tree- that is not the live one. When instructed by + static-master-run, we update the symlink and remove the old tree. + + static-mirror-run rsyncs either -current-push or -current-live for a component. + + LOCKING: + during all of static-mirror-run, we keep an exclusive lock on the + dir, i.e., the directory that holds tree-[ab] and cur. + +- static-mirror-run-all + + Run static-mirror-run for all components on this mirror, fetching the -live- tree. + + LOCKING: + none, but see static-mirror-run. + +- staticsync-ssh-wrap + + wrapper for ssh job dispatching on source, master, and mirror. + + LOCKING: + - on master, when syncing -live- trees: + a shared lock is held on .lock during + the rsync process. diff --git a/modules/staticsync/files/static-master-run b/modules/staticsync/files/static-master-run new file mode 100755 index 000000000..95c9355a5 --- /dev/null +++ b/modules/staticsync/files/static-master-run @@ -0,0 +1,252 @@ +#!/usr/bin/python + +import fcntl +import os +import shutil +import subprocess +import string +import sys +import tempfile +import time + +serialname = '.serial' +had_warnings = False + +conffile = '/etc/staticsync.conf' +config={} + +with open(conffile) as f: + for line in f: + line = line.rstrip() + if not line or line.startswith("#"): continue + (name, value) = line.split("=") + config[name] = value + +for key in ('base',): + if not key in config: + raise Exception("Configuration element '%s' not found in config file %s"%(key, conffile)) + +allclients = set() +with open('/etc/static-clients.conf') as f: + for line in f: + line = line.strip() + if line == "": continue + if line.startswith('#'): continue + allclients.add(line) + +def log(m): + t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime()) + print t, m + +def stage1(pipes, status, clients): + for c in clients: + p = pipes[c] + while 1: + line = p.stdout.readline() + if line == '': + status[c] = 'failed' + p.stdout.close() + p.stdin.close() + p.wait() + log("%s: failed with returncode %d"%(c,p.returncode)) + break + + line = line.strip() + log("%s >> %s"%(c, line)) + if not line.startswith('[MSM]'): continue + kw = string.split(line, ' ', 2)[1] + + if kw == 'ALREADY-CURRENT': + pipes[c].stdout.close() + pipes[c].stdin.close() + p.wait() + if p.returncode == 0: + log("%s: already current"%(c,)) + status[c] = 'ok' + else: + log("%s: said ALREADY-CURRENT but returncode %d"%(c,p.returncode)) + status[c] = 'failed' + break + elif kw == 'STAGE1-DONE': + log("%s: waiting"%(c,)) + status[c] = 'waiting' + break + elif kw in ['STAGE1-START']: + pass + else: + log("%s: ignoring unknown line"%(c,)) + +def count_statuses(status): + cnt = {} + for k in status: + v = status[k] + if v not in cnt: cnt[v] = 1 + else: cnt[v] += 1 + return cnt + +def stage2(pipes, status, command, clients): + for c in clients: + if status[c] != 'waiting': continue + log("%s << %s"%(c, command)) + pipes[c].stdin.write("%s\n"%(command,)) + + for c in clients: + if status[c] != 'waiting': continue + p = pipes[c] + + (o, dummy) = p.communicate('') + for l in string.split(o, "\n"): + log("%s >> %s"%(c, l)) + log("%s: returned %d"%(c, p.returncode)) + +def callout(component, serial, clients): + log("Calling clients...") + pipes = {} + status = {} + for c in clients: + args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)] + p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pipes[c] = p + status[c] = 'in-progress' + + log("Stage 1...") + stage1(pipes, status, clients) + log("Stage 1 done.") + cnt = count_statuses(status) + + if 'failed' in cnt and cnt['failed'] >= 2: + log("%d clients failed, aborting..."%(cnt['failed'],)) + stage2(pipes, status, 'abort', clients) + return False + + failedmirrorsfile = os.path.join(config['base'], 'master', component + "-failedmirrors") + if 'failed' in cnt: + log("WARNING: %d clients failed! Continuing anyway!"%(cnt['failed'],)) + global had_warnings + had_warnings = True + f = open(failedmirrorsfile, "w") + for c in status: + if status[c] == 'failed': f.write(c+"\n") + f.close() + else: + if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile) + + if 'waiting' in cnt: + log("Committing...") + stage2(pipes, status, 'go', clients) + return True + else: + log("All clients up to date.") + return True + +def load_component_info(component): + with open('/etc/static-components.conf') as f: + for line in f: + if line.startswith('#'): continue + field = line.strip().split() + if len(field) < 4: continue + if field[1] != component: continue + meta = {} + meta['master'] = field[0] + meta['sourcehost'] = field[2] + meta['sourcedir'] = field[3] + meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set() + meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set() + return meta + else: + return None + +cleanup_dirs = [] +def run_mirror(component): + meta = load_component_info(component) + if meta is None: + log("Component %s not found."%(component,)) + return False + clients = allclients - meta['extraignoreclients'] + + # setup + basemaster = os.path.join(config['base'], 'master') + componentdir = os.path.join(basemaster, component) + cur = componentdir + '-current-push' + live = componentdir + '-current-live' + tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new); + tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old); + os.chmod(tmpdir_new, 0755) + + locks = [] + lockfiles = [ os.path.join(basemaster, component + ".lock") ] + for p in lockfiles: + fd = os.open(p, os.O_RDONLY) + log("Acquiring lock for %s(%d)."%(p,fd)) + fcntl.flock(fd, fcntl.LOCK_EX) + locks.append(fd) + log("All locks acquired.") + + for p in (live, ): + if not os.path.exists(p): os.mkdir(p, 0755) + + #for p in (componentdir, live, tmpdir_new): + # if not os.path.exists(p): os.mkdir(p, 0755) + # fd = os.open(p, os.O_RDONLY) + # log("Acquiring lock for %s(%d)."%(p,fd)) + # fcntl.flock(fd, fcntl.LOCK_EX) + # locks.append(fd) + #log("All locks acquired.") + + serialfile = os.path.join(componentdir, serialname) + try: + with open(serialfile) as f: serial = int(f.read()) + except: + serial = int(time.time()) + with open(serialfile, "w") as f: f.write("%d\n"%(serial,)) + log("Serial is %s."%(serial,)) + + log("Populating %s."%(tmpdir_new,)) + subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new]) + + if os.path.exists(cur): + log("Removing existing %s."%(cur,)) + shutil.rmtree(cur) + + log("Renaming %s to %s."%(tmpdir_new, cur)) + os.rename(tmpdir_new, cur) + + proceed = callout(component, serial, clients) + + if proceed: + log("Moving %s aside."%(live,)) + os.rename(live, os.path.join(tmpdir_old, 'old')) + log("Renaming %s to %s."%(cur, live)) + os.rename(cur, live) + log("Cleaning up.") + shutil.rmtree(tmpdir_old) + if had_warnings: log("Done, with warnings.") + else: log("Done.") + ret = True + else: + log("Aborted.") + ret = False + + for fd in locks: + os.close(fd) + + return ret + + +if len(sys.argv) != 2: + print >> sys.stderr, "Usage: %s "%(sys.argv[0],) + sys.exit(1) +component = sys.argv[1] + +ok = False +try: + ok = run_mirror(component) +finally: + for p in cleanup_dirs: + if os.path.exists(p): shutil.rmtree(p) + +if not ok: + sys.exit(1) +# vim:set et: +# vim:set ts=2: +# vim:set shiftwidth=2: diff --git a/modules/staticsync/files/static-master-update-component b/modules/staticsync/files/static-master-update-component new file mode 100755 index 000000000..4ab5143e5 --- /dev/null +++ b/modules/staticsync/files/static-master-update-component @@ -0,0 +1,156 @@ +#!/bin/bash + +# Updates one component (i.e. subdirectory) in static-master/master + +# acquires a shared lock on the base directory (so that we know no updates are +# outgoing, as those acquire an exclusive one). Also acquired an exclusive lock +# on the component directory in question. +# +# The config file is a list of component source-directory pairs. + +# Copyright (c) 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +componentlist=/etc/static-components.conf +. /etc/staticsync.conf +if ! [ -n "$masterbase" ]; then + echo >&2 "masterbase not configured!" + exit 1 +fi + +set -e +set -u + +if [ "`id -u`" != "`stat -c %u "$masterbase"`" ]; then + echo >&2 "You are probably running this as the wrong user." + exit 1 +fi + +lock() { + local fd="$1"; shift + local path="$1"; shift + local exclusive="$1"; shift + + eval "exec $fd< '$path'" + + if [ "$exclusive" -gt 0 ]; then + locktype="-e" + else + locktype="-s" + fi + + if ! flock "$locktype" "$fd"; then + echo >&2 "$0: Cannot acquire lock on $path (flock $locktype failed) - Very bad, we should have waited!" + exit 1 + fi +} + +unlock() { + local fd="$1"; shift + + if ! flock -o "$fd"; then + echo >&2 "$0: Cannot release lock on fd $fd - This should not have happened!" + exit 1 + fi + eval "exec $fd<&-" +} + +if [ "$#" != 1 ]; then + echo >&2 "Usage: $0 " + exit 1 +fi + +component="$1" + +if [ "${component%/*}" != "$component" ] ; then + echo >&2 "$0: Invalid component: $component"; + exit 1 +fi + +srchost="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $3; exit}' "$componentlist")" +srcdir="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $4; exit}' "$componentlist")" +if [ -z "$srchost" ] || [ -z "$srcdir" ]; then + echo >&2 "$0: Invalid component: $component (not found in $componentlist)"; + exit 1 +fi + +tgtlock="$masterbase/$component.lock" +if ! [ -e "$tgtlock" ]; then + touch "$tgtlock" +fi +echo "$0: Acquiring lock on $tgtlock..." +lock 203 "$tgtlock" 1 + +tgt="$masterbase/$component" +if ! [ -d "$tgt" ]; then + echo "$0: Creating $tgt for $component"; + mkdir "$tgt" +fi + +if [ "$srchost" = "`hostname -f`" ]; then + src="$srcdir" +else + src="$srchost:$srcdir" +fi + +#echo "$0: Acquiring lock on $tgt..." +#lock 201 "$tgt" 1 + +tmpdir_new="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.incoming-XXXXXX")" +tmpdir_old="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.removing-XXXXXX")" +trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT +chmod 0755 "$tmpdir_new" + +#echo "$0: Acquiring lock on $tmpdir_new..." +#lock 202 "$tmpdir_new" 1 +echo "$0: Got them." + +echo "$0: Updating master copy of $component..." +rsync --delete \ + -trz \ + --links --hard-links --safe-links \ + --link-dest="$tgt" \ + --exclude='/.serial' \ + "$src/." "$tmpdir_new/." +echo "$0: Done. Committing." + +mv --no-target-directory "$tgt" "$tmpdir_old/old" +if ! mv --no-target-directory "$tmpdir_new" "$tgt"; then + echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt. Trying to recover" + rm -rf "$tgt" + mv --no-target-directory "$tmpdir_old/old" "$tgt" + echo >&2 "$0: Rolled back to old tree, maybe even successfully." + exit 1 +fi + +rm -rf "$tmpdir_new" "$tmpdir_old" +trap - EXIT + +date '+%s' > "$tgt/.serial" +#unlock 201 +#unlock 202 +unlock 203 +echo "$0: Triggering mirror runs..." +exec static-master-run "$component" + +# vim:set et: +# vim:set ts=2: +# vim:set shiftwidth=2: diff --git a/modules/staticsync/files/static-mirror-run b/modules/staticsync/files/static-mirror-run new file mode 100755 index 000000000..eac646bb3 --- /dev/null +++ b/modules/staticsync/files/static-mirror-run @@ -0,0 +1,178 @@ +#!/bin/bash + +# initiate a staged mirror update from sync-source for a component. +# +# if we have a serial file and we got a serial on the command line, only sync if the serial is different + +# Copyright (c) 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +set -e +set -u + +NAME="$(basename "$0")" + +usage() { + echo "Usage: $0 [--one-stage] []" +} + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then usage; exit 0; fi + +one_stage=0 +while :; do + case "${1:-}" in + --) + shift + break; + ;; + --one-stage) + shift + one_stage=1 + ;; + -*) usage >&2 + exit 1 + ;; + *) + break + ;; + esac +done + +COMPONENTDIR=${1:-}; shift +SYNC_SOURCE=${1:-}; shift +SYNC_SERIAL=${1:-}; shift || true +if [ -z "$COMPONENTDIR" ]; then usage >&2; exit 1; fi +if [ -z "$SYNC_SOURCE" ]; then usage >&2; exit 1; fi +COMPONENT="$(basename "${COMPONENTDIR}")" + +RSYNC="rsync" +RSYNC_BASE_OPTIONS="-rtvz --delete --links --hard-links --safe-links" +RSYNC_SSH_OPTIONS="ssh -o AddressFamily=inet -o BatchMode=yes" + +LOGDIR="$HOME/logs" +LOGFILE="$LOGDIR/$NAME-run-${COMPONENT}.log" + + +ALPHA="tree-a" +BRAVO="tree-b" +ACTIVE="cur" + +CNF_FILE="$HOME/etc/$NAME.conf" +! [ -e "$CNF_FILE" ] || . "$CNF_FILE" + +SOURCE="${SYNC_SOURCE}/" +COMPONENTDIR="${COMPONENTDIR}/" + +############################################### + +# point stdout and stderr to the logfile if it's not a tty. +# save stdout to fd5 for communications with the master + +log_setup() { + mkdir -p "$LOGDIR" + if ! [ -t 1 ]; then + # move current stdout to fd5 and reopen to logfile + exec 5>&1- + exec 1>> "$LOGFILE" + else + # duplicate stdout to fd5 + exec 5>&1 + fi + if ! [ -t 2 ]; then + exec 2>> "$LOGFILE" + fi +} + +log() { + echo "[$(date)][$NAME][$$] $1" +} + +lock() { + mkdir -p "$COMPONENTDIR" + exec 200< "$COMPONENTDIR" + if ! flock -e 200; then + log "Cannot acquire lock." + echo >&5 "[MSM] LOCK-ERROR" + exit 1 + fi + log "Got the lock." +} + +############################################### + + +log_setup +log "called with $*" +lock + +if [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" = "$ALPHA" ] ; then + staging="$BRAVO" + active="$ALPHA" +elif [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" != "$BRAVO" ] ; then + echo >&5 "Invalid state of ${COMPONENTDIR}${ACTIVE}." + exit 1 +else + staging="$ALPHA" + active="$BRAVO" +fi +log "active is $active; staging is $staging" + +rsync_source="${SOURCE}" +rsync_curactive="${COMPONENTDIR}${active}/" +rsync_target="${COMPONENTDIR}${staging}/" + +if [ -e "$rsync_curactive/.serial" ] && [ -n "$SYNC_SERIAL" ] && [ "$(cat $rsync_curactive/.serial)" = "$SYNC_SERIAL" ]; then + log "active is already at serial $SYNC_SERIAL. No action required." + echo >&5 "[MSM] ALREADY-CURRENT" + exit 0 +fi + +echo >&5 "[MSM] STAGE1-START" +log "Running $RSYNC $RSYNC_BASE_OPTIONS -e $RSYNC_SSH_OPTIONS --link-dest $rsync_curactive $rsync_source $rsync_target" +$RSYNC $RSYNC_BASE_OPTIONS -e "$RSYNC_SSH_OPTIONS" --link-dest "$rsync_curactive" "$rsync_source" "$rsync_target" +log "rsync done." +echo >&5 "[MSM] STAGE1-DONE" +if [ "$one_stage" -gt 0 ]; then + action="go" +else + read action +fi + +case "$action" in + go) + ln --symbolic --force --no-target-directory "$staging" "${COMPONENTDIR}$ACTIVE" + rm -rf "$rsync_curactive" + echo >&5 "[MSM] STAGE2-DONE" + log "stage2 done" + ;; + abort) + echo >&5 "[MSM] STAGE2-ABORT" + log "stage2 abort" + ;; + *) + echo >&5 "[MSM] STAGE2-UNKNOWN-ACTION $action" + log "stage2 unknown action $action" + exit 1 + ;; +esac + +savelog "$LOGFILE" diff --git a/modules/staticsync/files/static-mirror-run-all b/modules/staticsync/files/static-mirror-run-all new file mode 100755 index 000000000..c6aea0c10 --- /dev/null +++ b/modules/staticsync/files/static-mirror-run-all @@ -0,0 +1,46 @@ +#!/bin/bash + +# initiate a mirror run for all components on this host. + +# Copyright (c) 2012, 2015 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +set -u + +. /etc/staticsync.conf +if ! [ -n "$base" ]; then + echo >&2 "base not configured!" + exit 1 +fi + +awk -v host="$(hostname -f)" ' + !/^ *(#|$)/ { + split($6,ignorehosts,",") + for (i in ignorehosts) { + if (host == ignorehosts[i]) { + next + } + } + print $1, $2 + }' /etc/static-components.conf | + while read master component ; do + static-mirror-run --one-stage "$base/mirrors/$component" "$master:$component/-live-" + done diff --git a/modules/staticsync/files/static-update-component b/modules/staticsync/files/static-update-component new file mode 100755 index 000000000..455d17d38 --- /dev/null +++ b/modules/staticsync/files/static-update-component @@ -0,0 +1,89 @@ +#!/bin/bash + +# Copyright (c) 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +usage() { + echo >&2 "Usage: $0 " + exit 1 +} + +componentlist=/etc/static-components.conf +. /etc/staticsync.conf +if ! [ -n "$staticuser" ]; then + echo >&2 "staticuser not configured!" + exit 1 +fi + +if [ "$#" = 1 ]; then + component="$1" +else + usage +fi + + +if [ "${component%/*}" != "$component" ] ; then + echo >&2 "$0: Invalid component: $component"; + exit 1 +fi + +thishost=$(hostname -f) +masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$componentlist")" +srchost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $3; exit}' "$componentlist")" +srcdir="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $4; exit}' "$componentlist")" +inextralist="$( + awk -v component="$component" -v host="$thishost" ' + !/^ *(#|$)/ && $2 == component { + split($5,extra,",") + for (i in extra) { + if (host == extra[i]) { + printf "%s:%s\n", $3, $4 + exit + } + } + exit + }' "$componentlist" + )" +if [ -z "$srchost" ] || [ -z "$srcdir" ]; then + echo >&2 "$0: Invalid component: $component (not found in $componentlist)"; + exit 1 +fi + +if ! [ "$srchost" = "$thishost" ] && [ -z "$inextralist" ]; then + echo >&2 "Component $component is sourced from $srchost, and this host is neither that nor in the extra allowed list." + exit 1 +fi + +if [ "$srchost" = "$thishost" ] && ! [ -d "$srcdir" ]; then + echo >&2 "Component source directory $srcdir does not exist or is not a directory, or is not accessible." + exit 1 +fi + +if [ "`id -nu`" != "$staticuser" ]; then + sudo -u "$staticuser" static-update-component "$@" +else + ssh -o AddressFamily=inet -t -t -o ServerAliveInterval=300 -o PreferredAuthentications=publickey "$masterhost" static-master-update-component "$component" +fi + +# vim:set et: +# vim:set ts=2: +# vim:set shiftwidth=2: +# vim:set syn=sh: diff --git a/modules/staticsync/files/staticsync-ssh-wrap b/modules/staticsync/files/staticsync-ssh-wrap new file mode 100755 index 000000000..959b4f8f0 --- /dev/null +++ b/modules/staticsync/files/staticsync-ssh-wrap @@ -0,0 +1,225 @@ +#!/bin/bash + +# Copyright (c) 2009, 2010, 2012 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +set -e +set -u + +. /etc/staticsync.conf +if ! [ -n "$base" ]; then + echo >&2 "base not configured!" + exit 1 +fi +BASEDIR="$base" + +MYLOGNAME="`basename "$0"`[$$]" +COMPONENTLIST=/etc/static-components.conf + +usage() { + echo "local Usage: $0 " + echo "via ssh orig command:" + echo " mirror " + echo " rsync " + echo " static-master-update-component " +} + +one_more_arg() { + if [ "$#" -lt 1 ]; then + usage >&2 + exit 1 + fi +} + +info() { + logger -p daemon.info -t "$MYLOGNAME" "$1" +} + +croak() { + logger -s -p daemon.warn -t "$MYLOGNAME" "$1" + exit 1 +} + +do_mirror() { + local remote_host="$1"; shift + one_more_arg "$@" + local component="$1"; shift + one_more_arg "$@" + local serial="$1"; shift + + masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$COMPONENTLIST")" + if [ -z "$masterhost" ]; then + croak "Did not find master for component $component." + elif [ "$masterhost" != "$remote_host" ]; then + croak "$remote_host is not master for $component." + else + info "Host $remote_host triggered a mirror run for $component, serial $serial" + exec /usr/local/bin/static-mirror-run "$BASEDIR/mirrors/$component" "$remote_host:$component/-new-" "$serial" + echo >&2 "Exec failed" + croak "exec failed" + fi +} + +do_rsync_on_master() { + local remote_host="$1"; shift + local allowed_rsyncs + allowed_rsyncs=() + allowed_rsyncs+=("--server --sender -vlHtrze.iLsf --safe-links .") # wheezy + allowed_rsyncs+=("--server --sender -vlHtrze.iLsfx --safe-links .") # jessie + allowed_rsyncs+=("--server --sender -vlHtrze.iLsfxC --safe-links .") # stretch + + for cmd_idx in ${!allowed_rsyncs[*]}; do + args="${allowed_rsyncs[$cmd_idx]}" + for component in $(awk -v this_host="$(hostname -f)" '!/^ *(#|$)/ && $1 == this_host {print $2}' $COMPONENTLIST); do + if [ "$*" = "$args $component/-new-/" ] || [ "$*" = "$args ./$component/-new-/" ] ; then + local path="$BASEDIR/master/$component-current-push" + info "serving $remote_host with $path" + exec rsync $args "$path/." + croak "Exec failed" + elif [ "$*" = "$args $component/-live-/" ] || [ "$*" = "$args ./$component/-live-/" ] ; then + local path="$BASEDIR/master/$component-current-live" + info "host $remote_host wants $path, acquiring lock" + tgtlock="$BASEDIR/master/$component.lock" + if ! [ -e "$tgtlock" ]; then + touch "$tgtlock" + fi + exec 200< "$tgtlock" + if ! flock -s -w 0 200; then + echo >&2 "Cannot acquire shared lock on $tgtlock covering $path - this should mean an update is already underway anyway." + exit 1 + fi + exec rsync $args "$path/." + croak "Exec failed" + fi + done + done +} + +do_rsync_on_source() { + local remote_host="$1" + shift + + local allowed_rsyncs + allowed_rsyncs=() + + if [ -e "$COMPONENTLIST" ]; then + for path in $(awk -v host="$(hostname -f)" '!/^ *(#|$)/ && $3 == host {print $4}' $COMPONENTLIST); do + allowed_rsyncs+=("--server --sender -lHtrze.iLsf --safe-links . $path/.") # wheezy + allowed_rsyncs+=("--server --sender -lHtrze.iLsfx --safe-links . $path/.") # jessie + allowed_rsyncs+=("--server --sender -lHtrze.iLsfxC --safe-links . $path/.") # stretch + done + fi + for cmd_idx in ${!allowed_rsyncs[*]}; do + allowed="${allowed_rsyncs[$cmd_idx]}" + if [ "$*" = "$allowed" ]; then + info "Running for host $remote_host: rsync $*" + exec rsync "$@" + croak "Exec failed" + fi + done +} + +do_rsync() { + do_rsync_on_master "$@" + do_rsync_on_source "$@" + + info "NOT allowed for $remote_host: rsync $*" + echo >&2 "This rsync command ($@) not allowed." + exit 1 +} + +do_update_component() { + local remote_host="$1"; shift + + one_more_arg "$@" + component="$1" + shift + + hit="$( + awk -v this_host="$(hostname -f)" -v component="$component" -v host="$remote_host" ' + !/^ *(#|$)/ && $1 == this_host && $2 == component { + if ($3 == host) { + print $4 + exit + } + split($5,extra,",") + for (i in extra) { + if (host == extra[i]) { + printf "%s:%s\n", $3, $4 + exit + } + } + exit + }' "$COMPONENTLIST" + )" + if [ -n "$hit" ]; then + exec static-master-update-component "$component" + echo >&2 "Exec failed" + croak "exec failed" + else + info "Not whitelisted: $remote_host update $component" + echo >&2 "Not whitelisted: $remote_host update $component" + exit 1 + fi +} + + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 +fi + +one_more_arg "$@" +remote_host="$1" +shift + + +# check/parse remote command line +if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then + croak "Did not find SSH_ORIGINAL_COMMAND" +fi +set "dummy" ${SSH_ORIGINAL_COMMAND} +shift + +info "host $remote_host called with $*" + +one_more_arg "$@" +action="$1" +shift + +case "$action" in + # on a static mirror, update a component from its master + mirror) + do_mirror "$remote_host" "$@" + ;; + # on a static source, allow fetching from the master, + # on a master, allow fetching from a component's mirrors + rsync) + do_rsync "$remote_host" "$@" + ;; + # on a master, initiate an update of a component + static-master-update-component) + do_update_component "$remote_host" "$@" + ;; + *) + croak "Invalid operation '$action'" + ;; +esac diff --git a/modules/staticsync/manifests/base.pp b/modules/staticsync/manifests/base.pp new file mode 100644 index 000000000..89a0a383c --- /dev/null +++ b/modules/staticsync/manifests/base.pp @@ -0,0 +1,31 @@ +# the base class defining things common for all three static classes (master, mirror, source) +class roles::static::base { + $query = 'nodes[certname] { resources { type = "Class" and title = "Roles::Static_mirror" } }' + $static_mirrors = sort(puppetdb_query($query).map |$value| { $value["certname"] }) + + file { '/etc/static-components.conf': + content => template('roles/static-mirroring/static-components.conf.erb'), + } + + file { '/usr/local/bin/staticsync-ssh-wrap': + source => 'puppet:///modules/roles/static-mirroring/staticsync-ssh-wrap', + mode => '0555', + } + + file { '/usr/local/bin/static-update-component': + source => 'puppet:///modules/roles/static-mirroring/static-update-component', + mode => '0555', + } + + file { '/etc/staticsync.conf': + content => @("EOF"), + # This file is sourced by bash + # and parsed by python + # - empty lines and lines starting with a # are ignored. + # - other lines are key=value. No extra spaces anywhere. No quoting. + base=/srv/static.debian.org + masterbase=/home/staticsync/static-master/master + staticuser=staticsync + | EOF + } +} diff --git a/modules/staticsync/manifests/srvdir.pp b/modules/staticsync/manifests/srvdir.pp new file mode 100644 index 000000000..7fef3830c --- /dev/null +++ b/modules/staticsync/manifests/srvdir.pp @@ -0,0 +1,13 @@ +# create the directory on static hosts and disable backups +class roles::static::srvdir { + file { '/srv/static.debian.org': + ensure => directory, + mode => '0755', + owner => 'staticsync', + group => 'staticsync', + } + + file { '/srv/static.debian.org/.nobackup': + content => '', + } +} diff --git a/modules/staticsync/manifests/ssh.pp b/modules/staticsync/manifests/ssh.pp new file mode 100644 index 000000000..22078cb60 --- /dev/null +++ b/modules/staticsync/manifests/ssh.pp @@ -0,0 +1,20 @@ +# wrapper for ssh setup for statichosts +class roles::static::ssh( + Variant[Array[String], String] $add_tag, + String $collect_tag, + ) +{ + ssh::keygen {'staticsync': } + + ssh::authorized_key_add { 'staticsync': + target_user => 'staticsync', + command => "/usr/local/bin/staticsync-ssh-wrap ${::fqdn}", + key => $facts['staticsync_key'], + options => ['restrict', 'pty'], + collect_tag => $add_tag, + } + ssh::authorized_key_collect { 'staticsync': + target_user => 'staticsync', + collect_tag => $collect_tag, + } +} diff --git a/modules/staticsync/manifests/static_master.pp b/modules/staticsync/manifests/static_master.pp new file mode 100644 index 000000000..6a36ed3fc --- /dev/null +++ b/modules/staticsync/manifests/static_master.pp @@ -0,0 +1,38 @@ +# static master +# +# each component defines exactly one static master. Content is copied from the source host +# to the master, and from there to all the mirrors. +# +class roles::static_master { + include roles::static::base + include roles::static::srvdir + + # masters need to talk to mirrors and sources and themselves + class { 'roles::static::ssh': + add_tag => [ 'staticsync-mirror', 'staticsync-source', 'staticsync-master' ], + collect_tag => 'staticsync-master', + } + + file { '/usr/local/bin/static-master-run': + source => 'puppet:///modules/roles/static-mirroring/static-master-run', + mode => '0555', + } + file { '/usr/local/bin/static-master-update-component': + source => 'puppet:///modules/roles/static-mirroring/static-master-update-component', + mode => '0555', + } + file { '/etc/static-clients.conf': + content => template('roles/static-mirroring/static-clients.conf.erb'), + } + + file { '/home/staticsync/static-master': + ensure => link, + target => '/srv/static.debian.org', + } + file { '/srv/static.debian.org/master': + ensure => directory, + mode => '0755', + owner => 'staticsync', + group => 'staticsync', + } +} diff --git a/modules/staticsync/manifests/static_mirror.pp b/modules/staticsync/manifests/static_mirror.pp new file mode 100644 index 000000000..380ad3dac --- /dev/null +++ b/modules/staticsync/manifests/static_mirror.pp @@ -0,0 +1,31 @@ +# a static mirror +# +# this receives pushes from the master and then usually serves the content to the public +class roles::static_mirror { + include roles::static::base + include roles::static::srvdir + + # mirrors talk only to masters + class { 'roles::static::ssh': + add_tag => 'staticsync-master', + collect_tag => 'staticsync-mirror', + } + + file { '/usr/local/bin/static-mirror-run': + source => 'puppet:///modules/roles/static-mirroring/static-mirror-run', + mode => '0555', + } + + file { '/usr/local/bin/static-mirror-run-all': + source => 'puppet:///modules/roles/static-mirroring/static-mirror-run-all', + mode => '0555', + } + + file { '/etc/cron.d/puppet-static-mirror': ensure => absent, } + concat::fragment { 'puppet-crontab--static-mirror': + target => '/etc/cron.d/puppet-crontab', + content => @(EOF) + @reboot staticsync sleep 60; chronic static-mirror-run-all + | EOF + } +} diff --git a/modules/staticsync/manifests/static_source.pp b/modules/staticsync/manifests/static_source.pp new file mode 100644 index 000000000..960366cf9 --- /dev/null +++ b/modules/staticsync/manifests/static_source.pp @@ -0,0 +1,19 @@ +# a static source +# +# origin of static content. From here it goes to the static master before that one pushes it to the mirrors +class roles::static_source { + include roles::static::base + + if ! defined(Class["roles::static_master"]) { + # sources talk only to masters, but only set this up if we are not + # *also* a static master since we cannot have two meaningful roles::static:ssh + # instances in the current setup. + # + # this adds the limitation that the master of any component whose source is also a + # master node needs to have that same host as its master and not some other one. + class { 'roles::static::ssh': + add_tag => 'staticsync-master', + collect_tag => 'staticsync-source', + } + } +} diff --git a/modules/staticsync/templates/static-clients.conf.erb b/modules/staticsync/templates/static-clients.conf.erb new file mode 100644 index 000000000..6d2cd5bfe --- /dev/null +++ b/modules/staticsync/templates/static-clients.conf.erb @@ -0,0 +1,15 @@ +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## + +<%= + +# do not include mirrors in static_mirror_nopush +static_mirror_nopush = scope.lookupvar('deprecated::roles')['static_mirror_nopush'] + +scope.lookupvar('deprecated::roles')['static_mirror'].reject{ |x| static_mirror_nopush.include?(x) }.join("\n") + +# vim:set et: +# vim:set sts=4 ts=4: +# vim:set shiftwidth=4: +%> diff --git a/modules/staticsync/templates/static-components.conf.erb b/modules/staticsync/templates/static-components.conf.erb new file mode 100644 index 000000000..c04a3a56d --- /dev/null +++ b/modules/staticsync/templates/static-components.conf.erb @@ -0,0 +1,66 @@ +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## + +<%= + +lines = [] + +lines << "# This file has been autogenerated and pushed by puppet. Edit static-components.yaml in puppet." +lines << "# " + +# this is the list of static mirrors, or, technically, the list of +# nodes with the roles::static_mirror class applied to it. this should +# be populated from outside the template from PuppetDB, see: +# modules/roles/manifests/static_base.pp +mirrors = @static_mirrors + +config = YAML.load(File.open('/etc/puppet/modules/roles/misc/static-components.yaml').read) + + +config['mirrors'].each do |mirror, mc| + if not mirrors.include?(mirror) + fail("static-components.yaml defines mirror #{mirror} but we do not know of it") + end +end + +config['components'].each_pair do |component_name, component_conf| + %w{exclude-mirrors extra-push limit-mirrors}.each do |key| + component_conf[key] = [] unless component_conf.has_key?(key) + end + + srchost, srcpath = component_conf['source'].split(':', 2) + + if not component_conf['exclude-mirrors'].empty? and \ + not component_conf['limit-mirrors'].empty? then + fail("Component #{component_name} specifies both exclude-mirrors and limit-mirrors.") + end + + # In the end, we care about an exclude list, so let's invert limit-mirror and turn it into an exclude list + if not component_conf['limit-mirrors'].empty? then + component_conf['exclude-mirrors'] = mirrors.select do |m| + not component_conf['limit-mirrors'].include?(m) + end + end + + # mirrors may also specify limits as components-include (thus excluding all others). Apply this + config['mirrors'].each do |mirror, mc| + mirror_components_include = mc.fetch('components-include', []) + next if mirror_components_include.empty? + + if not mirror_components_include.include?(component_name) + next if component_conf['exclude-mirrors'].include?(mirror) # do not exclude twice + component_conf['exclude-mirrors'] << mirror + end + end + + exclude = component_conf['exclude-mirrors'].sort().join(',') + exclude = '-' if exclude == "" + extrapush = component_conf['extra-push'].sort().join(',') + extrapush = '-' if extrapush == "" + + lines << "#{component_conf['master']} #{component_name} #{srchost} #{srcpath} #{extrapush} #{exclude}" +end +lines.join("\n") +%>