+++ /dev/null
-
-- static-update-component is run by the user on the source host.
- If not run under sudo as the staticuser already, it sudos to the staticuser,
- re-execing itself. It them sshs to the static-master for that component to
- run static-master-update-component.
-
- LOCKING:
- none, but see static-master-update-component
-
-- static-master-update-component is run on the static-master.
- It rsyncs the contents from the source host to the static master, and then
- triggers static-master-run to push the content to the mirrors.
-
- The sync happens to a new <component>-updating.incoming-XXXXXX directory. On
- sync success, <component> is replaced with that new tree, and the
- static-master-run trigger happens.
-
- LOCKING:
- - exclusive locks are held on
- - <component>.lock
-
-- static-master-run triggers all the mirrors for a component to initiate syncs.
- When all mirrors have an up-to-date tree, they are instructed to update
- the cur-> symlink to the new tree.
-
- To begin with, static-master-run copies <component> to <component>-current-push.
- This is the tree all the mirrors then sync from. If the push was successful,
- <component>-current-push is renamed to <component>-current-live.
-
- LOCKING:
- - exclusive locks are held on
- - <component>.lock
-
-- static-mirror-run runs on a mirror and syncs components.
- There is a symlink called 'cur' that points to either tree-a or tree-b for
- each component. the cur tree is the one that is live, the other one usually
- does not exist, except when a sync is ongoing (or a previous one failed and
- we keep a partial tree).
-
- During a sync, we sync to the tree-<X> that is not the live one. When instructed by
- static-master-run, we update the symlink and remove the old tree.
-
- static-mirror-run rsyncs either -current-push or -current-live for a component.
-
- LOCKING:
- during all of static-mirror-run, we keep an exclusive lock on the <component>
- dir, i.e., the directory that holds tree-[ab] and cur.
-
-- static-mirror-run-all
-
- Run static-mirror-run for all components on this mirror, fetching the -live- tree.
-
- LOCKING:
- none, but see static-mirror-run.
-
-- staticsync-ssh-wrap
-
- wrapper for ssh job dispatching on source, master, and mirror.
-
- LOCKING:
- - on master, when syncing -live- trees:
- a shared lock is held on <component>.lock during
- the rsync process.
+++ /dev/null
-#!/usr/bin/python
-
-import fcntl
-import os
-import shutil
-import subprocess
-import string
-import sys
-import tempfile
-import time
-
-serialname = '.serial'
-had_warnings = False
-
-conffile = '/etc/staticsync.conf'
-config={}
-
-with open(conffile) as f:
- for line in f:
- line = line.rstrip()
- if not line or line.startswith("#"): continue
- (name, value) = line.split("=")
- config[name] = value
-
-for key in ('base',):
- if not key in config:
- raise Exception("Configuration element '%s' not found in config file %s"%(key, conffile))
-
-allclients = set()
-with open('/etc/static-clients.conf') as f:
- for line in f:
- line = line.strip()
- if line == "": continue
- if line.startswith('#'): continue
- allclients.add(line)
-
-def log(m):
- t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
- print t, m
-
-def stage1(pipes, status, clients):
- for c in clients:
- p = pipes[c]
- while 1:
- line = p.stdout.readline()
- if line == '':
- status[c] = 'failed'
- p.stdout.close()
- p.stdin.close()
- p.wait()
- log("%s: failed with returncode %d"%(c,p.returncode))
- break
-
- line = line.strip()
- log("%s >> %s"%(c, line))
- if not line.startswith('[MSM]'): continue
- kw = string.split(line, ' ', 2)[1]
-
- if kw == 'ALREADY-CURRENT':
- pipes[c].stdout.close()
- pipes[c].stdin.close()
- p.wait()
- if p.returncode == 0:
- log("%s: already current"%(c,))
- status[c] = 'ok'
- else:
- log("%s: said ALREADY-CURRENT but returncode %d"%(c,p.returncode))
- status[c] = 'failed'
- break
- elif kw == 'STAGE1-DONE':
- log("%s: waiting"%(c,))
- status[c] = 'waiting'
- break
- elif kw in ['STAGE1-START']:
- pass
- else:
- log("%s: ignoring unknown line"%(c,))
-
-def count_statuses(status):
- cnt = {}
- for k in status:
- v = status[k]
- if v not in cnt: cnt[v] = 1
- else: cnt[v] += 1
- return cnt
-
-def stage2(pipes, status, command, clients):
- for c in clients:
- if status[c] != 'waiting': continue
- log("%s << %s"%(c, command))
- pipes[c].stdin.write("%s\n"%(command,))
-
- for c in clients:
- if status[c] != 'waiting': continue
- p = pipes[c]
-
- (o, dummy) = p.communicate('')
- for l in string.split(o, "\n"):
- log("%s >> %s"%(c, l))
- log("%s: returned %d"%(c, p.returncode))
-
-def callout(component, serial, clients):
- log("Calling clients...")
- pipes = {}
- status = {}
- for c in clients:
- args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)]
- p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- pipes[c] = p
- status[c] = 'in-progress'
-
- log("Stage 1...")
- stage1(pipes, status, clients)
- log("Stage 1 done.")
- cnt = count_statuses(status)
-
- if 'failed' in cnt and cnt['failed'] >= 2:
- log("%d clients failed, aborting..."%(cnt['failed'],))
- stage2(pipes, status, 'abort', clients)
- return False
-
- failedmirrorsfile = os.path.join(config['base'], 'master', component + "-failedmirrors")
- if 'failed' in cnt:
- log("WARNING: %d clients failed! Continuing anyway!"%(cnt['failed'],))
- global had_warnings
- had_warnings = True
- f = open(failedmirrorsfile, "w")
- for c in status:
- if status[c] == 'failed': f.write(c+"\n")
- f.close()
- else:
- if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
-
- if 'waiting' in cnt:
- log("Committing...")
- stage2(pipes, status, 'go', clients)
- return True
- else:
- log("All clients up to date.")
- return True
-
-def load_component_info(component):
- with open('/etc/static-components.conf') as f:
- for line in f:
- if line.startswith('#'): continue
- field = line.strip().split()
- if len(field) < 4: continue
- if field[1] != component: continue
- meta = {}
- meta['master'] = field[0]
- meta['sourcehost'] = field[2]
- meta['sourcedir'] = field[3]
- meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set()
- meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set()
- return meta
- else:
- return None
-
-cleanup_dirs = []
-def run_mirror(component):
- meta = load_component_info(component)
- if meta is None:
- log("Component %s not found."%(component,))
- return False
- clients = allclients - meta['extraignoreclients']
-
- # setup
- basemaster = os.path.join(config['base'], 'master')
- componentdir = os.path.join(basemaster, component)
- cur = componentdir + '-current-push'
- live = componentdir + '-current-live'
- tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
- tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
- os.chmod(tmpdir_new, 0755)
-
- locks = []
- lockfiles = [ os.path.join(basemaster, component + ".lock") ]
- for p in lockfiles:
- fd = os.open(p, os.O_RDONLY)
- log("Acquiring lock for %s(%d)."%(p,fd))
- fcntl.flock(fd, fcntl.LOCK_EX)
- locks.append(fd)
- log("All locks acquired.")
-
- for p in (live, ):
- if not os.path.exists(p): os.mkdir(p, 0755)
-
- #for p in (componentdir, live, tmpdir_new):
- # if not os.path.exists(p): os.mkdir(p, 0755)
- # fd = os.open(p, os.O_RDONLY)
- # log("Acquiring lock for %s(%d)."%(p,fd))
- # fcntl.flock(fd, fcntl.LOCK_EX)
- # locks.append(fd)
- #log("All locks acquired.")
-
- serialfile = os.path.join(componentdir, serialname)
- try:
- with open(serialfile) as f: serial = int(f.read())
- except:
- serial = int(time.time())
- with open(serialfile, "w") as f: f.write("%d\n"%(serial,))
- log("Serial is %s."%(serial,))
-
- log("Populating %s."%(tmpdir_new,))
- subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new])
-
- if os.path.exists(cur):
- log("Removing existing %s."%(cur,))
- shutil.rmtree(cur)
-
- log("Renaming %s to %s."%(tmpdir_new, cur))
- os.rename(tmpdir_new, cur)
-
- proceed = callout(component, serial, clients)
-
- if proceed:
- log("Moving %s aside."%(live,))
- os.rename(live, os.path.join(tmpdir_old, 'old'))
- log("Renaming %s to %s."%(cur, live))
- os.rename(cur, live)
- log("Cleaning up.")
- shutil.rmtree(tmpdir_old)
- if had_warnings: log("Done, with warnings.")
- else: log("Done.")
- ret = True
- else:
- log("Aborted.")
- ret = False
-
- for fd in locks:
- os.close(fd)
-
- return ret
-
-
-if len(sys.argv) != 2:
- print >> sys.stderr, "Usage: %s <component>"%(sys.argv[0],)
- sys.exit(1)
-component = sys.argv[1]
-
-ok = False
-try:
- ok = run_mirror(component)
-finally:
- for p in cleanup_dirs:
- if os.path.exists(p): shutil.rmtree(p)
-
-if not ok:
- sys.exit(1)
-# vim:set et:
-# vim:set ts=2:
-# vim:set shiftwidth=2:
+++ /dev/null
-#!/bin/bash
-
-# Updates one component (i.e. subdirectory) in static-master/master
-
-# acquires a shared lock on the base directory (so that we know no updates are
-# outgoing, as those acquire an exclusive one). Also acquired an exclusive lock
-# on the component directory in question.
-#
-# The config file is a list of component source-directory pairs.
-
-# Copyright (c) 2012 Peter Palfrader
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-componentlist=/etc/static-components.conf
-. /etc/staticsync.conf
-if ! [ -n "$masterbase" ]; then
- echo >&2 "masterbase not configured!"
- exit 1
-fi
-
-set -e
-set -u
-
-if [ "`id -u`" != "`stat -c %u "$masterbase"`" ]; then
- echo >&2 "You are probably running this as the wrong user."
- exit 1
-fi
-
-lock() {
- local fd="$1"; shift
- local path="$1"; shift
- local exclusive="$1"; shift
-
- eval "exec $fd< '$path'"
-
- if [ "$exclusive" -gt 0 ]; then
- locktype="-e"
- else
- locktype="-s"
- fi
-
- if ! flock "$locktype" "$fd"; then
- echo >&2 "$0: Cannot acquire lock on $path (flock $locktype failed) - Very bad, we should have waited!"
- exit 1
- fi
-}
-
-unlock() {
- local fd="$1"; shift
-
- if ! flock -o "$fd"; then
- echo >&2 "$0: Cannot release lock on fd $fd - This should not have happened!"
- exit 1
- fi
- eval "exec $fd<&-"
-}
-
-if [ "$#" != 1 ]; then
- echo >&2 "Usage: $0 <component>"
- exit 1
-fi
-
-component="$1"
-
-if [ "${component%/*}" != "$component" ] ; then
- echo >&2 "$0: Invalid component: $component";
- exit 1
-fi
-
-srchost="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $3; exit}' "$componentlist")"
-srcdir="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $4; exit}' "$componentlist")"
-if [ -z "$srchost" ] || [ -z "$srcdir" ]; then
- echo >&2 "$0: Invalid component: $component (not found in $componentlist)";
- exit 1
-fi
-
-tgtlock="$masterbase/$component.lock"
-if ! [ -e "$tgtlock" ]; then
- touch "$tgtlock"
-fi
-echo "$0: Acquiring lock on $tgtlock..."
-lock 203 "$tgtlock" 1
-
-tgt="$masterbase/$component"
-if ! [ -d "$tgt" ]; then
- echo "$0: Creating $tgt for $component";
- mkdir "$tgt"
-fi
-
-if [ "$srchost" = "`hostname -f`" ]; then
- src="$srcdir"
-else
- src="$srchost:$srcdir"
-fi
-
-#echo "$0: Acquiring lock on $tgt..."
-#lock 201 "$tgt" 1
-
-tmpdir_new="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.incoming-XXXXXX")"
-tmpdir_old="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.removing-XXXXXX")"
-trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT
-chmod 0755 "$tmpdir_new"
-
-#echo "$0: Acquiring lock on $tmpdir_new..."
-#lock 202 "$tmpdir_new" 1
-echo "$0: Got them."
-
-echo "$0: Updating master copy of $component..."
-rsync --delete \
- -trz \
- --links --hard-links --safe-links \
- --link-dest="$tgt" \
- --exclude='/.serial' \
- "$src/." "$tmpdir_new/."
-echo "$0: Done. Committing."
-
-mv --no-target-directory "$tgt" "$tmpdir_old/old"
-if ! mv --no-target-directory "$tmpdir_new" "$tgt"; then
- echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt. Trying to recover"
- rm -rf "$tgt"
- mv --no-target-directory "$tmpdir_old/old" "$tgt"
- echo >&2 "$0: Rolled back to old tree, maybe even successfully."
- exit 1
-fi
-
-rm -rf "$tmpdir_new" "$tmpdir_old"
-trap - EXIT
-
-date '+%s' > "$tgt/.serial"
-#unlock 201
-#unlock 202
-unlock 203
-echo "$0: Triggering mirror runs..."
-exec static-master-run "$component"
-
-# vim:set et:
-# vim:set ts=2:
-# vim:set shiftwidth=2:
+++ /dev/null
-#!/bin/bash
-
-# initiate a staged mirror update from sync-source for a component.
-#
-# if we have a serial file and we got a serial on the command line, only sync if the serial is different
-
-# Copyright (c) 2012 Peter Palfrader
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
-set -e
-set -u
-
-NAME="$(basename "$0")"
-
-usage() {
- echo "Usage: $0 [--one-stage] <componentdir> <sync-source> [<serial>]"
-}
-
-if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then usage; exit 0; fi
-
-one_stage=0
-while :; do
- case "${1:-}" in
- --)
- shift
- break;
- ;;
- --one-stage)
- shift
- one_stage=1
- ;;
- -*) usage >&2
- exit 1
- ;;
- *)
- break
- ;;
- esac
-done
-
-COMPONENTDIR=${1:-}; shift
-SYNC_SOURCE=${1:-}; shift
-SYNC_SERIAL=${1:-}; shift || true
-if [ -z "$COMPONENTDIR" ]; then usage >&2; exit 1; fi
-if [ -z "$SYNC_SOURCE" ]; then usage >&2; exit 1; fi
-COMPONENT="$(basename "${COMPONENTDIR}")"
-
-RSYNC="rsync"
-RSYNC_BASE_OPTIONS="-rtvz --delete --links --hard-links --safe-links"
-RSYNC_SSH_OPTIONS="ssh -o AddressFamily=inet -o BatchMode=yes"
-
-LOGDIR="$HOME/logs"
-LOGFILE="$LOGDIR/$NAME-run-${COMPONENT}.log"
-
-
-ALPHA="tree-a"
-BRAVO="tree-b"
-ACTIVE="cur"
-
-CNF_FILE="$HOME/etc/$NAME.conf"
-! [ -e "$CNF_FILE" ] || . "$CNF_FILE"
-
-SOURCE="${SYNC_SOURCE}/"
-COMPONENTDIR="${COMPONENTDIR}/"
-
-###############################################
-
-# point stdout and stderr to the logfile if it's not a tty.
-# save stdout to fd5 for communications with the master
-
-log_setup() {
- mkdir -p "$LOGDIR"
- if ! [ -t 1 ]; then
- # move current stdout to fd5 and reopen to logfile
- exec 5>&1-
- exec 1>> "$LOGFILE"
- else
- # duplicate stdout to fd5
- exec 5>&1
- fi
- if ! [ -t 2 ]; then
- exec 2>> "$LOGFILE"
- fi
-}
-
-log() {
- echo "[$(date)][$NAME][$$] $1"
-}
-
-lock() {
- mkdir -p "$COMPONENTDIR"
- exec 200< "$COMPONENTDIR"
- if ! flock -e 200; then
- log "Cannot acquire lock."
- echo >&5 "[MSM] LOCK-ERROR"
- exit 1
- fi
- log "Got the lock."
-}
-
-###############################################
-
-
-log_setup
-log "called with $*"
-lock
-
-if [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" = "$ALPHA" ] ; then
- staging="$BRAVO"
- active="$ALPHA"
-elif [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" != "$BRAVO" ] ; then
- echo >&5 "Invalid state of ${COMPONENTDIR}${ACTIVE}."
- exit 1
-else
- staging="$ALPHA"
- active="$BRAVO"
-fi
-log "active is $active; staging is $staging"
-
-rsync_source="${SOURCE}"
-rsync_curactive="${COMPONENTDIR}${active}/"
-rsync_target="${COMPONENTDIR}${staging}/"
-
-if [ -e "$rsync_curactive/.serial" ] && [ -n "$SYNC_SERIAL" ] && [ "$(cat $rsync_curactive/.serial)" = "$SYNC_SERIAL" ]; then
- log "active is already at serial $SYNC_SERIAL. No action required."
- echo >&5 "[MSM] ALREADY-CURRENT"
- exit 0
-fi
-
-echo >&5 "[MSM] STAGE1-START"
-log "Running $RSYNC $RSYNC_BASE_OPTIONS -e $RSYNC_SSH_OPTIONS --link-dest $rsync_curactive $rsync_source $rsync_target"
-$RSYNC $RSYNC_BASE_OPTIONS -e "$RSYNC_SSH_OPTIONS" --link-dest "$rsync_curactive" "$rsync_source" "$rsync_target"
-log "rsync done."
-echo >&5 "[MSM] STAGE1-DONE"
-if [ "$one_stage" -gt 0 ]; then
- action="go"
-else
- read action
-fi
-
-case "$action" in
- go)
- ln --symbolic --force --no-target-directory "$staging" "${COMPONENTDIR}$ACTIVE"
- rm -rf "$rsync_curactive"
- echo >&5 "[MSM] STAGE2-DONE"
- log "stage2 done"
- ;;
- abort)
- echo >&5 "[MSM] STAGE2-ABORT"
- log "stage2 abort"
- ;;
- *)
- echo >&5 "[MSM] STAGE2-UNKNOWN-ACTION $action"
- log "stage2 unknown action $action"
- exit 1
- ;;
-esac
-
-savelog "$LOGFILE"
+++ /dev/null
-#!/bin/bash
-
-# initiate a mirror run for all components on this host.
-
-# Copyright (c) 2012, 2015 Peter Palfrader
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-set -u
-
-. /etc/staticsync.conf
-if ! [ -n "$base" ]; then
- echo >&2 "base not configured!"
- exit 1
-fi
-
-awk -v host="$(hostname -f)" '
- !/^ *(#|$)/ {
- split($6,ignorehosts,",")
- for (i in ignorehosts) {
- if (host == ignorehosts[i]) {
- next
- }
- }
- print $1, $2
- }' /etc/static-components.conf |
- while read master component ; do
- static-mirror-run --one-stage "$base/mirrors/$component" "$master:$component/-live-"
- done
+++ /dev/null
-#!/bin/bash
-
-# Copyright (c) 2012 Peter Palfrader
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-usage() {
- echo >&2 "Usage: $0 <component>"
- exit 1
-}
-
-componentlist=/etc/static-components.conf
-. /etc/staticsync.conf
-if ! [ -n "$staticuser" ]; then
- echo >&2 "staticuser not configured!"
- exit 1
-fi
-
-if [ "$#" = 1 ]; then
- component="$1"
-else
- usage
-fi
-
-
-if [ "${component%/*}" != "$component" ] ; then
- echo >&2 "$0: Invalid component: $component";
- exit 1
-fi
-
-thishost=$(hostname -f)
-masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$componentlist")"
-srchost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $3; exit}' "$componentlist")"
-srcdir="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $4; exit}' "$componentlist")"
-inextralist="$(
- awk -v component="$component" -v host="$thishost" '
- !/^ *(#|$)/ && $2 == component {
- split($5,extra,",")
- for (i in extra) {
- if (host == extra[i]) {
- printf "%s:%s\n", $3, $4
- exit
- }
- }
- exit
- }' "$componentlist"
- )"
-if [ -z "$srchost" ] || [ -z "$srcdir" ]; then
- echo >&2 "$0: Invalid component: $component (not found in $componentlist)";
- exit 1
-fi
-
-if ! [ "$srchost" = "$thishost" ] && [ -z "$inextralist" ]; then
- echo >&2 "Component $component is sourced from $srchost, and this host is neither that nor in the extra allowed list."
- exit 1
-fi
-
-if [ "$srchost" = "$thishost" ] && ! [ -d "$srcdir" ]; then
- echo >&2 "Component source directory $srcdir does not exist or is not a directory, or is not accessible."
- exit 1
-fi
-
-if [ "`id -nu`" != "$staticuser" ]; then
- sudo -u "$staticuser" static-update-component "$@"
-else
- ssh -o AddressFamily=inet -t -t -o ServerAliveInterval=300 -o PreferredAuthentications=publickey "$masterhost" static-master-update-component "$component"
-fi
-
-# vim:set et:
-# vim:set ts=2:
-# vim:set shiftwidth=2:
-# vim:set syn=sh:
+++ /dev/null
-#!/bin/bash
-
-# Copyright (c) 2009, 2010, 2012 Peter Palfrader
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-set -e
-set -u
-
-. /etc/staticsync.conf
-if ! [ -n "$base" ]; then
- echo >&2 "base not configured!"
- exit 1
-fi
-BASEDIR="$base"
-
-MYLOGNAME="`basename "$0"`[$$]"
-COMPONENTLIST=/etc/static-components.conf
-
-usage() {
- echo "local Usage: $0 <host>"
- echo "via ssh orig command:"
- echo " mirror <component> <serial>"
- echo " rsync <stuff>"
- echo " static-master-update-component <component>"
-}
-
-one_more_arg() {
- if [ "$#" -lt 1 ]; then
- usage >&2
- exit 1
- fi
-}
-
-info() {
- logger -p daemon.info -t "$MYLOGNAME" "$1"
-}
-
-croak() {
- logger -s -p daemon.warn -t "$MYLOGNAME" "$1"
- exit 1
-}
-
-do_mirror() {
- local remote_host="$1"; shift
- one_more_arg "$@"
- local component="$1"; shift
- one_more_arg "$@"
- local serial="$1"; shift
-
- masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$COMPONENTLIST")"
- if [ -z "$masterhost" ]; then
- croak "Did not find master for component $component."
- elif [ "$masterhost" != "$remote_host" ]; then
- croak "$remote_host is not master for $component."
- else
- info "Host $remote_host triggered a mirror run for $component, serial $serial"
- exec /usr/local/bin/static-mirror-run "$BASEDIR/mirrors/$component" "$remote_host:$component/-new-" "$serial"
- echo >&2 "Exec failed"
- croak "exec failed"
- fi
-}
-
-do_rsync_on_master() {
- local remote_host="$1"; shift
- local allowed_rsyncs
- allowed_rsyncs=()
- allowed_rsyncs+=("--server --sender -vlHtrze.iLsf --safe-links .") # wheezy
- allowed_rsyncs+=("--server --sender -vlHtrze.iLsfx --safe-links .") # jessie
- allowed_rsyncs+=("--server --sender -vlHtrze.iLsfxC --safe-links .") # stretch
-
- for cmd_idx in ${!allowed_rsyncs[*]}; do
- args="${allowed_rsyncs[$cmd_idx]}"
- for component in $(awk -v this_host="$(hostname -f)" '!/^ *(#|$)/ && $1 == this_host {print $2}' $COMPONENTLIST); do
- if [ "$*" = "$args $component/-new-/" ] || [ "$*" = "$args ./$component/-new-/" ] ; then
- local path="$BASEDIR/master/$component-current-push"
- info "serving $remote_host with $path"
- exec rsync $args "$path/."
- croak "Exec failed"
- elif [ "$*" = "$args $component/-live-/" ] || [ "$*" = "$args ./$component/-live-/" ] ; then
- local path="$BASEDIR/master/$component-current-live"
- info "host $remote_host wants $path, acquiring lock"
- tgtlock="$BASEDIR/master/$component.lock"
- if ! [ -e "$tgtlock" ]; then
- touch "$tgtlock"
- fi
- exec 200< "$tgtlock"
- if ! flock -s -w 0 200; then
- echo >&2 "Cannot acquire shared lock on $tgtlock covering $path - this should mean an update is already underway anyway."
- exit 1
- fi
- exec rsync $args "$path/."
- croak "Exec failed"
- fi
- done
- done
-}
-
-do_rsync_on_source() {
- local remote_host="$1"
- shift
-
- local allowed_rsyncs
- allowed_rsyncs=()
-
- if [ -e "$COMPONENTLIST" ]; then
- for path in $(awk -v host="$(hostname -f)" '!/^ *(#|$)/ && $3 == host {print $4}' $COMPONENTLIST); do
- allowed_rsyncs+=("--server --sender -lHtrze.iLsf --safe-links . $path/.") # wheezy
- allowed_rsyncs+=("--server --sender -lHtrze.iLsfx --safe-links . $path/.") # jessie
- allowed_rsyncs+=("--server --sender -lHtrze.iLsfxC --safe-links . $path/.") # stretch
- done
- fi
- for cmd_idx in ${!allowed_rsyncs[*]}; do
- allowed="${allowed_rsyncs[$cmd_idx]}"
- if [ "$*" = "$allowed" ]; then
- info "Running for host $remote_host: rsync $*"
- exec rsync "$@"
- croak "Exec failed"
- fi
- done
-}
-
-do_rsync() {
- do_rsync_on_master "$@"
- do_rsync_on_source "$@"
-
- info "NOT allowed for $remote_host: rsync $*"
- echo >&2 "This rsync command ($@) not allowed."
- exit 1
-}
-
-do_update_component() {
- local remote_host="$1"; shift
-
- one_more_arg "$@"
- component="$1"
- shift
-
- hit="$(
- awk -v this_host="$(hostname -f)" -v component="$component" -v host="$remote_host" '
- !/^ *(#|$)/ && $1 == this_host && $2 == component {
- if ($3 == host) {
- print $4
- exit
- }
- split($5,extra,",")
- for (i in extra) {
- if (host == extra[i]) {
- printf "%s:%s\n", $3, $4
- exit
- }
- }
- exit
- }' "$COMPONENTLIST"
- )"
- if [ -n "$hit" ]; then
- exec static-master-update-component "$component"
- echo >&2 "Exec failed"
- croak "exec failed"
- else
- info "Not whitelisted: $remote_host update $component"
- echo >&2 "Not whitelisted: $remote_host update $component"
- exit 1
- fi
-}
-
-
-if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
- usage
- exit 0
-fi
-
-one_more_arg "$@"
-remote_host="$1"
-shift
-
-
-# check/parse remote command line
-if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then
- croak "Did not find SSH_ORIGINAL_COMMAND"
-fi
-set "dummy" ${SSH_ORIGINAL_COMMAND}
-shift
-
-info "host $remote_host called with $*"
-
-one_more_arg "$@"
-action="$1"
-shift
-
-case "$action" in
- # on a static mirror, update a component from its master
- mirror)
- do_mirror "$remote_host" "$@"
- ;;
- # on a static source, allow fetching from the master,
- # on a master, allow fetching from a component's mirrors
- rsync)
- do_rsync "$remote_host" "$@"
- ;;
- # on a master, initiate an update of a component
- static-master-update-component)
- do_update_component "$remote_host" "$@"
- ;;
- *)
- croak "Invalid operation '$action'"
- ;;
-esac
+++ /dev/null
-# the base class defining things common for all three static classes (master, mirror, source)
-class roles::static::base {
- $query = 'nodes[certname] { resources { type = "Class" and title = "Roles::Static_mirror" } }'
- $static_mirrors = sort(puppetdb_query($query).map |$value| { $value["certname"] })
-
- file { '/etc/static-components.conf':
- content => template('roles/static-mirroring/static-components.conf.erb'),
- }
-
- file { '/usr/local/bin/staticsync-ssh-wrap':
- source => 'puppet:///modules/roles/static-mirroring/staticsync-ssh-wrap',
- mode => '0555',
- }
-
- file { '/usr/local/bin/static-update-component':
- source => 'puppet:///modules/roles/static-mirroring/static-update-component',
- mode => '0555',
- }
-
- file { '/etc/staticsync.conf':
- content => @("EOF"),
- # This file is sourced by bash
- # and parsed by python
- # - empty lines and lines starting with a # are ignored.
- # - other lines are key=value. No extra spaces anywhere. No quoting.
- base=/srv/static.debian.org
- masterbase=/home/staticsync/static-master/master
- staticuser=staticsync
- | EOF
- }
-}
+++ /dev/null
-# create the directory on static hosts and disable backups
-class roles::static::srvdir {
- file { '/srv/static.debian.org':
- ensure => directory,
- mode => '0755',
- owner => 'staticsync',
- group => 'staticsync',
- }
-
- file { '/srv/static.debian.org/.nobackup':
- content => '',
- }
-}
+++ /dev/null
-# wrapper for ssh setup for statichosts
-class roles::static::ssh(
- Variant[Array[String], String] $add_tag,
- String $collect_tag,
- )
-{
- ssh::keygen {'staticsync': }
-
- ssh::authorized_key_add { 'staticsync':
- target_user => 'staticsync',
- command => "/usr/local/bin/staticsync-ssh-wrap ${::fqdn}",
- key => $facts['staticsync_key'],
- options => ['restrict', 'pty'],
- collect_tag => $add_tag,
- }
- ssh::authorized_key_collect { 'staticsync':
- target_user => 'staticsync',
- collect_tag => $collect_tag,
- }
-}
+++ /dev/null
-# static master
-#
-# each component defines exactly one static master. Content is copied from the source host
-# to the master, and from there to all the mirrors.
-#
-class roles::static_master {
- include roles::static::base
- include roles::static::srvdir
-
- # masters need to talk to mirrors and sources and themselves
- class { 'roles::static::ssh':
- add_tag => [ 'staticsync-mirror', 'staticsync-source', 'staticsync-master' ],
- collect_tag => 'staticsync-master',
- }
-
- file { '/usr/local/bin/static-master-run':
- source => 'puppet:///modules/roles/static-mirroring/static-master-run',
- mode => '0555',
- }
- file { '/usr/local/bin/static-master-update-component':
- source => 'puppet:///modules/roles/static-mirroring/static-master-update-component',
- mode => '0555',
- }
- file { '/etc/static-clients.conf':
- content => template('roles/static-mirroring/static-clients.conf.erb'),
- }
-
- file { '/home/staticsync/static-master':
- ensure => link,
- target => '/srv/static.debian.org',
- }
- file { '/srv/static.debian.org/master':
- ensure => directory,
- mode => '0755',
- owner => 'staticsync',
- group => 'staticsync',
- }
-}
+++ /dev/null
-# a static mirror
-#
-# this receives pushes from the master and then usually serves the content to the public
-class roles::static_mirror {
- include roles::static::base
- include roles::static::srvdir
-
- # mirrors talk only to masters
- class { 'roles::static::ssh':
- add_tag => 'staticsync-master',
- collect_tag => 'staticsync-mirror',
- }
-
- file { '/usr/local/bin/static-mirror-run':
- source => 'puppet:///modules/roles/static-mirroring/static-mirror-run',
- mode => '0555',
- }
-
- file { '/usr/local/bin/static-mirror-run-all':
- source => 'puppet:///modules/roles/static-mirroring/static-mirror-run-all',
- mode => '0555',
- }
-
- file { '/etc/cron.d/puppet-static-mirror': ensure => absent, }
- concat::fragment { 'puppet-crontab--static-mirror':
- target => '/etc/cron.d/puppet-crontab',
- content => @(EOF)
- @reboot staticsync sleep 60; chronic static-mirror-run-all
- | EOF
- }
-}
+++ /dev/null
-# a static source
-#
-# origin of static content. From here it goes to the static master before that one pushes it to the mirrors
-class roles::static_source {
- include roles::static::base
-
- if ! defined(Class["roles::static_master"]) {
- # sources talk only to masters, but only set this up if we are not
- # *also* a static master since we cannot have two meaningful roles::static:ssh
- # instances in the current setup.
- #
- # this adds the limitation that the master of any component whose source is also a
- # master node needs to have that same host as its master and not some other one.
- class { 'roles::static::ssh':
- add_tag => 'staticsync-master',
- collect_tag => 'staticsync-source',
- }
- }
-}
+++ /dev/null
-##
-## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
-##
-
-<%=
-
-# do not include mirrors in static_mirror_nopush
-static_mirror_nopush = scope.lookupvar('deprecated::roles')['static_mirror_nopush']
-
-scope.lookupvar('deprecated::roles')['static_mirror'].reject{ |x| static_mirror_nopush.include?(x) }.join("\n")
-
-# vim:set et:
-# vim:set sts=4 ts=4:
-# vim:set shiftwidth=4:
-%>
+++ /dev/null
-##
-## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
-## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git
-##
-
-<%=
-
-lines = []
-
-lines << "# This file has been autogenerated and pushed by puppet. Edit static-components.yaml in puppet."
-lines << "# <master> <service> <source host> <directory> <extra push hosts, comma separated> <hosts to not mirror this component to>"
-
-# this is the list of static mirrors, or, technically, the list of
-# nodes with the roles::static_mirror class applied to it. this should
-# be populated from outside the template from PuppetDB, see:
-# modules/roles/manifests/static_base.pp
-mirrors = @static_mirrors
-
-config = YAML.load(File.open('/etc/puppet/modules/roles/misc/static-components.yaml').read)
-
-
-config['mirrors'].each do |mirror, mc|
- if not mirrors.include?(mirror)
- fail("static-components.yaml defines mirror #{mirror} but we do not know of it")
- end
-end
-
-config['components'].each_pair do |component_name, component_conf|
- %w{exclude-mirrors extra-push limit-mirrors}.each do |key|
- component_conf[key] = [] unless component_conf.has_key?(key)
- end
-
- srchost, srcpath = component_conf['source'].split(':', 2)
-
- if not component_conf['exclude-mirrors'].empty? and \
- not component_conf['limit-mirrors'].empty? then
- fail("Component #{component_name} specifies both exclude-mirrors and limit-mirrors.")
- end
-
- # In the end, we care about an exclude list, so let's invert limit-mirror and turn it into an exclude list
- if not component_conf['limit-mirrors'].empty? then
- component_conf['exclude-mirrors'] = mirrors.select do |m|
- not component_conf['limit-mirrors'].include?(m)
- end
- end
-
- # mirrors may also specify limits as components-include (thus excluding all others). Apply this
- config['mirrors'].each do |mirror, mc|
- mirror_components_include = mc.fetch('components-include', [])
- next if mirror_components_include.empty?
-
- if not mirror_components_include.include?(component_name)
- next if component_conf['exclude-mirrors'].include?(mirror) # do not exclude twice
- component_conf['exclude-mirrors'] << mirror
- end
- end
-
- exclude = component_conf['exclude-mirrors'].sort().join(',')
- exclude = '-' if exclude == ""
- extrapush = component_conf['extra-push'].sort().join(',')
- extrapush = '-' if extrapush == ""
-
- lines << "#{component_conf['master']} #{component_name} #{srchost} #{srcpath} #{extrapush} #{exclude}"
-end
-lines.join("\n")
-%>
--- /dev/null
+
+- static-update-component is run by the user on the source host.
+ If not run under sudo as the staticuser already, it sudos to the staticuser,
+ re-execing itself. It them sshs to the static-master for that component to
+ run static-master-update-component.
+
+ LOCKING:
+ none, but see static-master-update-component
+
+- static-master-update-component is run on the static-master.
+ It rsyncs the contents from the source host to the static master, and then
+ triggers static-master-run to push the content to the mirrors.
+
+ The sync happens to a new <component>-updating.incoming-XXXXXX directory. On
+ sync success, <component> is replaced with that new tree, and the
+ static-master-run trigger happens.
+
+ LOCKING:
+ - exclusive locks are held on
+ - <component>.lock
+
+- static-master-run triggers all the mirrors for a component to initiate syncs.
+ When all mirrors have an up-to-date tree, they are instructed to update
+ the cur-> symlink to the new tree.
+
+ To begin with, static-master-run copies <component> to <component>-current-push.
+ This is the tree all the mirrors then sync from. If the push was successful,
+ <component>-current-push is renamed to <component>-current-live.
+
+ LOCKING:
+ - exclusive locks are held on
+ - <component>.lock
+
+- static-mirror-run runs on a mirror and syncs components.
+ There is a symlink called 'cur' that points to either tree-a or tree-b for
+ each component. the cur tree is the one that is live, the other one usually
+ does not exist, except when a sync is ongoing (or a previous one failed and
+ we keep a partial tree).
+
+ During a sync, we sync to the tree-<X> that is not the live one. When instructed by
+ static-master-run, we update the symlink and remove the old tree.
+
+ static-mirror-run rsyncs either -current-push or -current-live for a component.
+
+ LOCKING:
+ during all of static-mirror-run, we keep an exclusive lock on the <component>
+ dir, i.e., the directory that holds tree-[ab] and cur.
+
+- static-mirror-run-all
+
+ Run static-mirror-run for all components on this mirror, fetching the -live- tree.
+
+ LOCKING:
+ none, but see static-mirror-run.
+
+- staticsync-ssh-wrap
+
+ wrapper for ssh job dispatching on source, master, and mirror.
+
+ LOCKING:
+ - on master, when syncing -live- trees:
+ a shared lock is held on <component>.lock during
+ the rsync process.
--- /dev/null
+#!/usr/bin/python
+
+import fcntl
+import os
+import shutil
+import subprocess
+import string
+import sys
+import tempfile
+import time
+
+serialname = '.serial'
+had_warnings = False
+
+conffile = '/etc/staticsync.conf'
+config={}
+
+with open(conffile) as f:
+ for line in f:
+ line = line.rstrip()
+ if not line or line.startswith("#"): continue
+ (name, value) = line.split("=")
+ config[name] = value
+
+for key in ('base',):
+ if not key in config:
+ raise Exception("Configuration element '%s' not found in config file %s"%(key, conffile))
+
+allclients = set()
+with open('/etc/static-clients.conf') as f:
+ for line in f:
+ line = line.strip()
+ if line == "": continue
+ if line.startswith('#'): continue
+ allclients.add(line)
+
+def log(m):
+ t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
+ print t, m
+
+def stage1(pipes, status, clients):
+ for c in clients:
+ p = pipes[c]
+ while 1:
+ line = p.stdout.readline()
+ if line == '':
+ status[c] = 'failed'
+ p.stdout.close()
+ p.stdin.close()
+ p.wait()
+ log("%s: failed with returncode %d"%(c,p.returncode))
+ break
+
+ line = line.strip()
+ log("%s >> %s"%(c, line))
+ if not line.startswith('[MSM]'): continue
+ kw = string.split(line, ' ', 2)[1]
+
+ if kw == 'ALREADY-CURRENT':
+ pipes[c].stdout.close()
+ pipes[c].stdin.close()
+ p.wait()
+ if p.returncode == 0:
+ log("%s: already current"%(c,))
+ status[c] = 'ok'
+ else:
+ log("%s: said ALREADY-CURRENT but returncode %d"%(c,p.returncode))
+ status[c] = 'failed'
+ break
+ elif kw == 'STAGE1-DONE':
+ log("%s: waiting"%(c,))
+ status[c] = 'waiting'
+ break
+ elif kw in ['STAGE1-START']:
+ pass
+ else:
+ log("%s: ignoring unknown line"%(c,))
+
+def count_statuses(status):
+ cnt = {}
+ for k in status:
+ v = status[k]
+ if v not in cnt: cnt[v] = 1
+ else: cnt[v] += 1
+ return cnt
+
+def stage2(pipes, status, command, clients):
+ for c in clients:
+ if status[c] != 'waiting': continue
+ log("%s << %s"%(c, command))
+ pipes[c].stdin.write("%s\n"%(command,))
+
+ for c in clients:
+ if status[c] != 'waiting': continue
+ p = pipes[c]
+
+ (o, dummy) = p.communicate('')
+ for l in string.split(o, "\n"):
+ log("%s >> %s"%(c, l))
+ log("%s: returned %d"%(c, p.returncode))
+
+def callout(component, serial, clients):
+ log("Calling clients...")
+ pipes = {}
+ status = {}
+ for c in clients:
+ args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)]
+ p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ pipes[c] = p
+ status[c] = 'in-progress'
+
+ log("Stage 1...")
+ stage1(pipes, status, clients)
+ log("Stage 1 done.")
+ cnt = count_statuses(status)
+
+ if 'failed' in cnt and cnt['failed'] >= 2:
+ log("%d clients failed, aborting..."%(cnt['failed'],))
+ stage2(pipes, status, 'abort', clients)
+ return False
+
+ failedmirrorsfile = os.path.join(config['base'], 'master', component + "-failedmirrors")
+ if 'failed' in cnt:
+ log("WARNING: %d clients failed! Continuing anyway!"%(cnt['failed'],))
+ global had_warnings
+ had_warnings = True
+ f = open(failedmirrorsfile, "w")
+ for c in status:
+ if status[c] == 'failed': f.write(c+"\n")
+ f.close()
+ else:
+ if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
+
+ if 'waiting' in cnt:
+ log("Committing...")
+ stage2(pipes, status, 'go', clients)
+ return True
+ else:
+ log("All clients up to date.")
+ return True
+
+def load_component_info(component):
+ with open('/etc/static-components.conf') as f:
+ for line in f:
+ if line.startswith('#'): continue
+ field = line.strip().split()
+ if len(field) < 4: continue
+ if field[1] != component: continue
+ meta = {}
+ meta['master'] = field[0]
+ meta['sourcehost'] = field[2]
+ meta['sourcedir'] = field[3]
+ meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set()
+ meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set()
+ return meta
+ else:
+ return None
+
+cleanup_dirs = []
+def run_mirror(component):
+ meta = load_component_info(component)
+ if meta is None:
+ log("Component %s not found."%(component,))
+ return False
+ clients = allclients - meta['extraignoreclients']
+
+ # setup
+ basemaster = os.path.join(config['base'], 'master')
+ componentdir = os.path.join(basemaster, component)
+ cur = componentdir + '-current-push'
+ live = componentdir + '-current-live'
+ tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
+ tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
+ os.chmod(tmpdir_new, 0755)
+
+ locks = []
+ lockfiles = [ os.path.join(basemaster, component + ".lock") ]
+ for p in lockfiles:
+ fd = os.open(p, os.O_RDONLY)
+ log("Acquiring lock for %s(%d)."%(p,fd))
+ fcntl.flock(fd, fcntl.LOCK_EX)
+ locks.append(fd)
+ log("All locks acquired.")
+
+ for p in (live, ):
+ if not os.path.exists(p): os.mkdir(p, 0755)
+
+ #for p in (componentdir, live, tmpdir_new):
+ # if not os.path.exists(p): os.mkdir(p, 0755)
+ # fd = os.open(p, os.O_RDONLY)
+ # log("Acquiring lock for %s(%d)."%(p,fd))
+ # fcntl.flock(fd, fcntl.LOCK_EX)
+ # locks.append(fd)
+ #log("All locks acquired.")
+
+ serialfile = os.path.join(componentdir, serialname)
+ try:
+ with open(serialfile) as f: serial = int(f.read())
+ except:
+ serial = int(time.time())
+ with open(serialfile, "w") as f: f.write("%d\n"%(serial,))
+ log("Serial is %s."%(serial,))
+
+ log("Populating %s."%(tmpdir_new,))
+ subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new])
+
+ if os.path.exists(cur):
+ log("Removing existing %s."%(cur,))
+ shutil.rmtree(cur)
+
+ log("Renaming %s to %s."%(tmpdir_new, cur))
+ os.rename(tmpdir_new, cur)
+
+ proceed = callout(component, serial, clients)
+
+ if proceed:
+ log("Moving %s aside."%(live,))
+ os.rename(live, os.path.join(tmpdir_old, 'old'))
+ log("Renaming %s to %s."%(cur, live))
+ os.rename(cur, live)
+ log("Cleaning up.")
+ shutil.rmtree(tmpdir_old)
+ if had_warnings: log("Done, with warnings.")
+ else: log("Done.")
+ ret = True
+ else:
+ log("Aborted.")
+ ret = False
+
+ for fd in locks:
+ os.close(fd)
+
+ return ret
+
+
+if len(sys.argv) != 2:
+ print >> sys.stderr, "Usage: %s <component>"%(sys.argv[0],)
+ sys.exit(1)
+component = sys.argv[1]
+
+ok = False
+try:
+ ok = run_mirror(component)
+finally:
+ for p in cleanup_dirs:
+ if os.path.exists(p): shutil.rmtree(p)
+
+if not ok:
+ sys.exit(1)
+# vim:set et:
+# vim:set ts=2:
+# vim:set shiftwidth=2:
--- /dev/null
+#!/bin/bash
+
+# Updates one component (i.e. subdirectory) in static-master/master
+
+# acquires a shared lock on the base directory (so that we know no updates are
+# outgoing, as those acquire an exclusive one). Also acquired an exclusive lock
+# on the component directory in question.
+#
+# The config file is a list of component source-directory pairs.
+
+# Copyright (c) 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+componentlist=/etc/static-components.conf
+. /etc/staticsync.conf
+if ! [ -n "$masterbase" ]; then
+ echo >&2 "masterbase not configured!"
+ exit 1
+fi
+
+set -e
+set -u
+
+if [ "`id -u`" != "`stat -c %u "$masterbase"`" ]; then
+ echo >&2 "You are probably running this as the wrong user."
+ exit 1
+fi
+
+lock() {
+ local fd="$1"; shift
+ local path="$1"; shift
+ local exclusive="$1"; shift
+
+ eval "exec $fd< '$path'"
+
+ if [ "$exclusive" -gt 0 ]; then
+ locktype="-e"
+ else
+ locktype="-s"
+ fi
+
+ if ! flock "$locktype" "$fd"; then
+ echo >&2 "$0: Cannot acquire lock on $path (flock $locktype failed) - Very bad, we should have waited!"
+ exit 1
+ fi
+}
+
+unlock() {
+ local fd="$1"; shift
+
+ if ! flock -o "$fd"; then
+ echo >&2 "$0: Cannot release lock on fd $fd - This should not have happened!"
+ exit 1
+ fi
+ eval "exec $fd<&-"
+}
+
+if [ "$#" != 1 ]; then
+ echo >&2 "Usage: $0 <component>"
+ exit 1
+fi
+
+component="$1"
+
+if [ "${component%/*}" != "$component" ] ; then
+ echo >&2 "$0: Invalid component: $component";
+ exit 1
+fi
+
+srchost="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $3; exit}' "$componentlist")"
+srcdir="$(awk -v this_host="$(hostname -f)" -v component="$component" '!/^ *(#|$)/ && $1 == this_host && $2 == component {print $4; exit}' "$componentlist")"
+if [ -z "$srchost" ] || [ -z "$srcdir" ]; then
+ echo >&2 "$0: Invalid component: $component (not found in $componentlist)";
+ exit 1
+fi
+
+tgtlock="$masterbase/$component.lock"
+if ! [ -e "$tgtlock" ]; then
+ touch "$tgtlock"
+fi
+echo "$0: Acquiring lock on $tgtlock..."
+lock 203 "$tgtlock" 1
+
+tgt="$masterbase/$component"
+if ! [ -d "$tgt" ]; then
+ echo "$0: Creating $tgt for $component";
+ mkdir "$tgt"
+fi
+
+if [ "$srchost" = "`hostname -f`" ]; then
+ src="$srcdir"
+else
+ src="$srchost:$srcdir"
+fi
+
+#echo "$0: Acquiring lock on $tgt..."
+#lock 201 "$tgt" 1
+
+tmpdir_new="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.incoming-XXXXXX")"
+tmpdir_old="$(mktemp -d --tmpdir="$masterbase" "${component}-updating.removing-XXXXXX")"
+trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT
+chmod 0755 "$tmpdir_new"
+
+#echo "$0: Acquiring lock on $tmpdir_new..."
+#lock 202 "$tmpdir_new" 1
+echo "$0: Got them."
+
+echo "$0: Updating master copy of $component..."
+rsync --delete \
+ -trz \
+ --links --hard-links --safe-links \
+ --link-dest="$tgt" \
+ --exclude='/.serial' \
+ "$src/." "$tmpdir_new/."
+echo "$0: Done. Committing."
+
+mv --no-target-directory "$tgt" "$tmpdir_old/old"
+if ! mv --no-target-directory "$tmpdir_new" "$tgt"; then
+ echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt. Trying to recover"
+ rm -rf "$tgt"
+ mv --no-target-directory "$tmpdir_old/old" "$tgt"
+ echo >&2 "$0: Rolled back to old tree, maybe even successfully."
+ exit 1
+fi
+
+rm -rf "$tmpdir_new" "$tmpdir_old"
+trap - EXIT
+
+date '+%s' > "$tgt/.serial"
+#unlock 201
+#unlock 202
+unlock 203
+echo "$0: Triggering mirror runs..."
+exec static-master-run "$component"
+
+# vim:set et:
+# vim:set ts=2:
+# vim:set shiftwidth=2:
--- /dev/null
+#!/bin/bash
+
+# initiate a staged mirror update from sync-source for a component.
+#
+# if we have a serial file and we got a serial on the command line, only sync if the serial is different
+
+# Copyright (c) 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+set -e
+set -u
+
+NAME="$(basename "$0")"
+
+usage() {
+ echo "Usage: $0 [--one-stage] <componentdir> <sync-source> [<serial>]"
+}
+
+if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then usage; exit 0; fi
+
+one_stage=0
+while :; do
+ case "${1:-}" in
+ --)
+ shift
+ break;
+ ;;
+ --one-stage)
+ shift
+ one_stage=1
+ ;;
+ -*) usage >&2
+ exit 1
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+COMPONENTDIR=${1:-}; shift
+SYNC_SOURCE=${1:-}; shift
+SYNC_SERIAL=${1:-}; shift || true
+if [ -z "$COMPONENTDIR" ]; then usage >&2; exit 1; fi
+if [ -z "$SYNC_SOURCE" ]; then usage >&2; exit 1; fi
+COMPONENT="$(basename "${COMPONENTDIR}")"
+
+RSYNC="rsync"
+RSYNC_BASE_OPTIONS="-rtvz --delete --links --hard-links --safe-links"
+RSYNC_SSH_OPTIONS="ssh -o AddressFamily=inet -o BatchMode=yes"
+
+LOGDIR="$HOME/logs"
+LOGFILE="$LOGDIR/$NAME-run-${COMPONENT}.log"
+
+
+ALPHA="tree-a"
+BRAVO="tree-b"
+ACTIVE="cur"
+
+CNF_FILE="$HOME/etc/$NAME.conf"
+! [ -e "$CNF_FILE" ] || . "$CNF_FILE"
+
+SOURCE="${SYNC_SOURCE}/"
+COMPONENTDIR="${COMPONENTDIR}/"
+
+###############################################
+
+# point stdout and stderr to the logfile if it's not a tty.
+# save stdout to fd5 for communications with the master
+
+log_setup() {
+ mkdir -p "$LOGDIR"
+ if ! [ -t 1 ]; then
+ # move current stdout to fd5 and reopen to logfile
+ exec 5>&1-
+ exec 1>> "$LOGFILE"
+ else
+ # duplicate stdout to fd5
+ exec 5>&1
+ fi
+ if ! [ -t 2 ]; then
+ exec 2>> "$LOGFILE"
+ fi
+}
+
+log() {
+ echo "[$(date)][$NAME][$$] $1"
+}
+
+lock() {
+ mkdir -p "$COMPONENTDIR"
+ exec 200< "$COMPONENTDIR"
+ if ! flock -e 200; then
+ log "Cannot acquire lock."
+ echo >&5 "[MSM] LOCK-ERROR"
+ exit 1
+ fi
+ log "Got the lock."
+}
+
+###############################################
+
+
+log_setup
+log "called with $*"
+lock
+
+if [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" = "$ALPHA" ] ; then
+ staging="$BRAVO"
+ active="$ALPHA"
+elif [ -e "${COMPONENTDIR}${ACTIVE}" ] && [ "$(readlink "${COMPONENTDIR}${ACTIVE}")" != "$BRAVO" ] ; then
+ echo >&5 "Invalid state of ${COMPONENTDIR}${ACTIVE}."
+ exit 1
+else
+ staging="$ALPHA"
+ active="$BRAVO"
+fi
+log "active is $active; staging is $staging"
+
+rsync_source="${SOURCE}"
+rsync_curactive="${COMPONENTDIR}${active}/"
+rsync_target="${COMPONENTDIR}${staging}/"
+
+if [ -e "$rsync_curactive/.serial" ] && [ -n "$SYNC_SERIAL" ] && [ "$(cat $rsync_curactive/.serial)" = "$SYNC_SERIAL" ]; then
+ log "active is already at serial $SYNC_SERIAL. No action required."
+ echo >&5 "[MSM] ALREADY-CURRENT"
+ exit 0
+fi
+
+echo >&5 "[MSM] STAGE1-START"
+log "Running $RSYNC $RSYNC_BASE_OPTIONS -e $RSYNC_SSH_OPTIONS --link-dest $rsync_curactive $rsync_source $rsync_target"
+$RSYNC $RSYNC_BASE_OPTIONS -e "$RSYNC_SSH_OPTIONS" --link-dest "$rsync_curactive" "$rsync_source" "$rsync_target"
+log "rsync done."
+echo >&5 "[MSM] STAGE1-DONE"
+if [ "$one_stage" -gt 0 ]; then
+ action="go"
+else
+ read action
+fi
+
+case "$action" in
+ go)
+ ln --symbolic --force --no-target-directory "$staging" "${COMPONENTDIR}$ACTIVE"
+ rm -rf "$rsync_curactive"
+ echo >&5 "[MSM] STAGE2-DONE"
+ log "stage2 done"
+ ;;
+ abort)
+ echo >&5 "[MSM] STAGE2-ABORT"
+ log "stage2 abort"
+ ;;
+ *)
+ echo >&5 "[MSM] STAGE2-UNKNOWN-ACTION $action"
+ log "stage2 unknown action $action"
+ exit 1
+ ;;
+esac
+
+savelog "$LOGFILE"
--- /dev/null
+#!/bin/bash
+
+# initiate a mirror run for all components on this host.
+
+# Copyright (c) 2012, 2015 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+set -u
+
+. /etc/staticsync.conf
+if ! [ -n "$base" ]; then
+ echo >&2 "base not configured!"
+ exit 1
+fi
+
+awk -v host="$(hostname -f)" '
+ !/^ *(#|$)/ {
+ split($6,ignorehosts,",")
+ for (i in ignorehosts) {
+ if (host == ignorehosts[i]) {
+ next
+ }
+ }
+ print $1, $2
+ }' /etc/static-components.conf |
+ while read master component ; do
+ static-mirror-run --one-stage "$base/mirrors/$component" "$master:$component/-live-"
+ done
--- /dev/null
+#!/bin/bash
+
+# Copyright (c) 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+usage() {
+ echo >&2 "Usage: $0 <component>"
+ exit 1
+}
+
+componentlist=/etc/static-components.conf
+. /etc/staticsync.conf
+if ! [ -n "$staticuser" ]; then
+ echo >&2 "staticuser not configured!"
+ exit 1
+fi
+
+if [ "$#" = 1 ]; then
+ component="$1"
+else
+ usage
+fi
+
+
+if [ "${component%/*}" != "$component" ] ; then
+ echo >&2 "$0: Invalid component: $component";
+ exit 1
+fi
+
+thishost=$(hostname -f)
+masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$componentlist")"
+srchost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $3; exit}' "$componentlist")"
+srcdir="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $4; exit}' "$componentlist")"
+inextralist="$(
+ awk -v component="$component" -v host="$thishost" '
+ !/^ *(#|$)/ && $2 == component {
+ split($5,extra,",")
+ for (i in extra) {
+ if (host == extra[i]) {
+ printf "%s:%s\n", $3, $4
+ exit
+ }
+ }
+ exit
+ }' "$componentlist"
+ )"
+if [ -z "$srchost" ] || [ -z "$srcdir" ]; then
+ echo >&2 "$0: Invalid component: $component (not found in $componentlist)";
+ exit 1
+fi
+
+if ! [ "$srchost" = "$thishost" ] && [ -z "$inextralist" ]; then
+ echo >&2 "Component $component is sourced from $srchost, and this host is neither that nor in the extra allowed list."
+ exit 1
+fi
+
+if [ "$srchost" = "$thishost" ] && ! [ -d "$srcdir" ]; then
+ echo >&2 "Component source directory $srcdir does not exist or is not a directory, or is not accessible."
+ exit 1
+fi
+
+if [ "`id -nu`" != "$staticuser" ]; then
+ sudo -u "$staticuser" static-update-component "$@"
+else
+ ssh -o AddressFamily=inet -t -t -o ServerAliveInterval=300 -o PreferredAuthentications=publickey "$masterhost" static-master-update-component "$component"
+fi
+
+# vim:set et:
+# vim:set ts=2:
+# vim:set shiftwidth=2:
+# vim:set syn=sh:
--- /dev/null
+#!/bin/bash
+
+# Copyright (c) 2009, 2010, 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+set -e
+set -u
+
+. /etc/staticsync.conf
+if ! [ -n "$base" ]; then
+ echo >&2 "base not configured!"
+ exit 1
+fi
+BASEDIR="$base"
+
+MYLOGNAME="`basename "$0"`[$$]"
+COMPONENTLIST=/etc/static-components.conf
+
+usage() {
+ echo "local Usage: $0 <host>"
+ echo "via ssh orig command:"
+ echo " mirror <component> <serial>"
+ echo " rsync <stuff>"
+ echo " static-master-update-component <component>"
+}
+
+one_more_arg() {
+ if [ "$#" -lt 1 ]; then
+ usage >&2
+ exit 1
+ fi
+}
+
+info() {
+ logger -p daemon.info -t "$MYLOGNAME" "$1"
+}
+
+croak() {
+ logger -s -p daemon.warn -t "$MYLOGNAME" "$1"
+ exit 1
+}
+
+do_mirror() {
+ local remote_host="$1"; shift
+ one_more_arg "$@"
+ local component="$1"; shift
+ one_more_arg "$@"
+ local serial="$1"; shift
+
+ masterhost="$(awk -v component="$component" '!/^ *(#|$)/ && $2 == component {print $1; exit}' "$COMPONENTLIST")"
+ if [ -z "$masterhost" ]; then
+ croak "Did not find master for component $component."
+ elif [ "$masterhost" != "$remote_host" ]; then
+ croak "$remote_host is not master for $component."
+ else
+ info "Host $remote_host triggered a mirror run for $component, serial $serial"
+ exec /usr/local/bin/static-mirror-run "$BASEDIR/mirrors/$component" "$remote_host:$component/-new-" "$serial"
+ echo >&2 "Exec failed"
+ croak "exec failed"
+ fi
+}
+
+do_rsync_on_master() {
+ local remote_host="$1"; shift
+ local allowed_rsyncs
+ allowed_rsyncs=()
+ allowed_rsyncs+=("--server --sender -vlHtrze.iLsf --safe-links .") # wheezy
+ allowed_rsyncs+=("--server --sender -vlHtrze.iLsfx --safe-links .") # jessie
+ allowed_rsyncs+=("--server --sender -vlHtrze.iLsfxC --safe-links .") # stretch
+
+ for cmd_idx in ${!allowed_rsyncs[*]}; do
+ args="${allowed_rsyncs[$cmd_idx]}"
+ for component in $(awk -v this_host="$(hostname -f)" '!/^ *(#|$)/ && $1 == this_host {print $2}' $COMPONENTLIST); do
+ if [ "$*" = "$args $component/-new-/" ] || [ "$*" = "$args ./$component/-new-/" ] ; then
+ local path="$BASEDIR/master/$component-current-push"
+ info "serving $remote_host with $path"
+ exec rsync $args "$path/."
+ croak "Exec failed"
+ elif [ "$*" = "$args $component/-live-/" ] || [ "$*" = "$args ./$component/-live-/" ] ; then
+ local path="$BASEDIR/master/$component-current-live"
+ info "host $remote_host wants $path, acquiring lock"
+ tgtlock="$BASEDIR/master/$component.lock"
+ if ! [ -e "$tgtlock" ]; then
+ touch "$tgtlock"
+ fi
+ exec 200< "$tgtlock"
+ if ! flock -s -w 0 200; then
+ echo >&2 "Cannot acquire shared lock on $tgtlock covering $path - this should mean an update is already underway anyway."
+ exit 1
+ fi
+ exec rsync $args "$path/."
+ croak "Exec failed"
+ fi
+ done
+ done
+}
+
+do_rsync_on_source() {
+ local remote_host="$1"
+ shift
+
+ local allowed_rsyncs
+ allowed_rsyncs=()
+
+ if [ -e "$COMPONENTLIST" ]; then
+ for path in $(awk -v host="$(hostname -f)" '!/^ *(#|$)/ && $3 == host {print $4}' $COMPONENTLIST); do
+ allowed_rsyncs+=("--server --sender -lHtrze.iLsf --safe-links . $path/.") # wheezy
+ allowed_rsyncs+=("--server --sender -lHtrze.iLsfx --safe-links . $path/.") # jessie
+ allowed_rsyncs+=("--server --sender -lHtrze.iLsfxC --safe-links . $path/.") # stretch
+ done
+ fi
+ for cmd_idx in ${!allowed_rsyncs[*]}; do
+ allowed="${allowed_rsyncs[$cmd_idx]}"
+ if [ "$*" = "$allowed" ]; then
+ info "Running for host $remote_host: rsync $*"
+ exec rsync "$@"
+ croak "Exec failed"
+ fi
+ done
+}
+
+do_rsync() {
+ do_rsync_on_master "$@"
+ do_rsync_on_source "$@"
+
+ info "NOT allowed for $remote_host: rsync $*"
+ echo >&2 "This rsync command ($@) not allowed."
+ exit 1
+}
+
+do_update_component() {
+ local remote_host="$1"; shift
+
+ one_more_arg "$@"
+ component="$1"
+ shift
+
+ hit="$(
+ awk -v this_host="$(hostname -f)" -v component="$component" -v host="$remote_host" '
+ !/^ *(#|$)/ && $1 == this_host && $2 == component {
+ if ($3 == host) {
+ print $4
+ exit
+ }
+ split($5,extra,",")
+ for (i in extra) {
+ if (host == extra[i]) {
+ printf "%s:%s\n", $3, $4
+ exit
+ }
+ }
+ exit
+ }' "$COMPONENTLIST"
+ )"
+ if [ -n "$hit" ]; then
+ exec static-master-update-component "$component"
+ echo >&2 "Exec failed"
+ croak "exec failed"
+ else
+ info "Not whitelisted: $remote_host update $component"
+ echo >&2 "Not whitelisted: $remote_host update $component"
+ exit 1
+ fi
+}
+
+
+if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
+ usage
+ exit 0
+fi
+
+one_more_arg "$@"
+remote_host="$1"
+shift
+
+
+# check/parse remote command line
+if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then
+ croak "Did not find SSH_ORIGINAL_COMMAND"
+fi
+set "dummy" ${SSH_ORIGINAL_COMMAND}
+shift
+
+info "host $remote_host called with $*"
+
+one_more_arg "$@"
+action="$1"
+shift
+
+case "$action" in
+ # on a static mirror, update a component from its master
+ mirror)
+ do_mirror "$remote_host" "$@"
+ ;;
+ # on a static source, allow fetching from the master,
+ # on a master, allow fetching from a component's mirrors
+ rsync)
+ do_rsync "$remote_host" "$@"
+ ;;
+ # on a master, initiate an update of a component
+ static-master-update-component)
+ do_update_component "$remote_host" "$@"
+ ;;
+ *)
+ croak "Invalid operation '$action'"
+ ;;
+esac
--- /dev/null
+# the base class defining things common for all three static classes (master, mirror, source)
+class roles::static::base {
+ $query = 'nodes[certname] { resources { type = "Class" and title = "Roles::Static_mirror" } }'
+ $static_mirrors = sort(puppetdb_query($query).map |$value| { $value["certname"] })
+
+ file { '/etc/static-components.conf':
+ content => template('roles/static-mirroring/static-components.conf.erb'),
+ }
+
+ file { '/usr/local/bin/staticsync-ssh-wrap':
+ source => 'puppet:///modules/roles/static-mirroring/staticsync-ssh-wrap',
+ mode => '0555',
+ }
+
+ file { '/usr/local/bin/static-update-component':
+ source => 'puppet:///modules/roles/static-mirroring/static-update-component',
+ mode => '0555',
+ }
+
+ file { '/etc/staticsync.conf':
+ content => @("EOF"),
+ # This file is sourced by bash
+ # and parsed by python
+ # - empty lines and lines starting with a # are ignored.
+ # - other lines are key=value. No extra spaces anywhere. No quoting.
+ base=/srv/static.debian.org
+ masterbase=/home/staticsync/static-master/master
+ staticuser=staticsync
+ | EOF
+ }
+}
--- /dev/null
+# create the directory on static hosts and disable backups
+class roles::static::srvdir {
+ file { '/srv/static.debian.org':
+ ensure => directory,
+ mode => '0755',
+ owner => 'staticsync',
+ group => 'staticsync',
+ }
+
+ file { '/srv/static.debian.org/.nobackup':
+ content => '',
+ }
+}
--- /dev/null
+# wrapper for ssh setup for statichosts
+class roles::static::ssh(
+ Variant[Array[String], String] $add_tag,
+ String $collect_tag,
+ )
+{
+ ssh::keygen {'staticsync': }
+
+ ssh::authorized_key_add { 'staticsync':
+ target_user => 'staticsync',
+ command => "/usr/local/bin/staticsync-ssh-wrap ${::fqdn}",
+ key => $facts['staticsync_key'],
+ options => ['restrict', 'pty'],
+ collect_tag => $add_tag,
+ }
+ ssh::authorized_key_collect { 'staticsync':
+ target_user => 'staticsync',
+ collect_tag => $collect_tag,
+ }
+}
--- /dev/null
+# static master
+#
+# each component defines exactly one static master. Content is copied from the source host
+# to the master, and from there to all the mirrors.
+#
+class roles::static_master {
+ include roles::static::base
+ include roles::static::srvdir
+
+ # masters need to talk to mirrors and sources and themselves
+ class { 'roles::static::ssh':
+ add_tag => [ 'staticsync-mirror', 'staticsync-source', 'staticsync-master' ],
+ collect_tag => 'staticsync-master',
+ }
+
+ file { '/usr/local/bin/static-master-run':
+ source => 'puppet:///modules/roles/static-mirroring/static-master-run',
+ mode => '0555',
+ }
+ file { '/usr/local/bin/static-master-update-component':
+ source => 'puppet:///modules/roles/static-mirroring/static-master-update-component',
+ mode => '0555',
+ }
+ file { '/etc/static-clients.conf':
+ content => template('roles/static-mirroring/static-clients.conf.erb'),
+ }
+
+ file { '/home/staticsync/static-master':
+ ensure => link,
+ target => '/srv/static.debian.org',
+ }
+ file { '/srv/static.debian.org/master':
+ ensure => directory,
+ mode => '0755',
+ owner => 'staticsync',
+ group => 'staticsync',
+ }
+}
--- /dev/null
+# a static mirror
+#
+# this receives pushes from the master and then usually serves the content to the public
+class roles::static_mirror {
+ include roles::static::base
+ include roles::static::srvdir
+
+ # mirrors talk only to masters
+ class { 'roles::static::ssh':
+ add_tag => 'staticsync-master',
+ collect_tag => 'staticsync-mirror',
+ }
+
+ file { '/usr/local/bin/static-mirror-run':
+ source => 'puppet:///modules/roles/static-mirroring/static-mirror-run',
+ mode => '0555',
+ }
+
+ file { '/usr/local/bin/static-mirror-run-all':
+ source => 'puppet:///modules/roles/static-mirroring/static-mirror-run-all',
+ mode => '0555',
+ }
+
+ file { '/etc/cron.d/puppet-static-mirror': ensure => absent, }
+ concat::fragment { 'puppet-crontab--static-mirror':
+ target => '/etc/cron.d/puppet-crontab',
+ content => @(EOF)
+ @reboot staticsync sleep 60; chronic static-mirror-run-all
+ | EOF
+ }
+}
--- /dev/null
+# a static source
+#
+# origin of static content. From here it goes to the static master before that one pushes it to the mirrors
+class roles::static_source {
+ include roles::static::base
+
+ if ! defined(Class["roles::static_master"]) {
+ # sources talk only to masters, but only set this up if we are not
+ # *also* a static master since we cannot have two meaningful roles::static:ssh
+ # instances in the current setup.
+ #
+ # this adds the limitation that the master of any component whose source is also a
+ # master node needs to have that same host as its master and not some other one.
+ class { 'roles::static::ssh':
+ add_tag => 'staticsync-master',
+ collect_tag => 'staticsync-source',
+ }
+ }
+}
--- /dev/null
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+##
+
+<%=
+
+# do not include mirrors in static_mirror_nopush
+static_mirror_nopush = scope.lookupvar('deprecated::roles')['static_mirror_nopush']
+
+scope.lookupvar('deprecated::roles')['static_mirror'].reject{ |x| static_mirror_nopush.include?(x) }.join("\n")
+
+# vim:set et:
+# vim:set sts=4 ts=4:
+# vim:set shiftwidth=4:
+%>
--- /dev/null
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git
+##
+
+<%=
+
+lines = []
+
+lines << "# This file has been autogenerated and pushed by puppet. Edit static-components.yaml in puppet."
+lines << "# <master> <service> <source host> <directory> <extra push hosts, comma separated> <hosts to not mirror this component to>"
+
+# this is the list of static mirrors, or, technically, the list of
+# nodes with the roles::static_mirror class applied to it. this should
+# be populated from outside the template from PuppetDB, see:
+# modules/roles/manifests/static_base.pp
+mirrors = @static_mirrors
+
+config = YAML.load(File.open('/etc/puppet/modules/roles/misc/static-components.yaml').read)
+
+
+config['mirrors'].each do |mirror, mc|
+ if not mirrors.include?(mirror)
+ fail("static-components.yaml defines mirror #{mirror} but we do not know of it")
+ end
+end
+
+config['components'].each_pair do |component_name, component_conf|
+ %w{exclude-mirrors extra-push limit-mirrors}.each do |key|
+ component_conf[key] = [] unless component_conf.has_key?(key)
+ end
+
+ srchost, srcpath = component_conf['source'].split(':', 2)
+
+ if not component_conf['exclude-mirrors'].empty? and \
+ not component_conf['limit-mirrors'].empty? then
+ fail("Component #{component_name} specifies both exclude-mirrors and limit-mirrors.")
+ end
+
+ # In the end, we care about an exclude list, so let's invert limit-mirror and turn it into an exclude list
+ if not component_conf['limit-mirrors'].empty? then
+ component_conf['exclude-mirrors'] = mirrors.select do |m|
+ not component_conf['limit-mirrors'].include?(m)
+ end
+ end
+
+ # mirrors may also specify limits as components-include (thus excluding all others). Apply this
+ config['mirrors'].each do |mirror, mc|
+ mirror_components_include = mc.fetch('components-include', [])
+ next if mirror_components_include.empty?
+
+ if not mirror_components_include.include?(component_name)
+ next if component_conf['exclude-mirrors'].include?(mirror) # do not exclude twice
+ component_conf['exclude-mirrors'] << mirror
+ end
+ end
+
+ exclude = component_conf['exclude-mirrors'].sort().join(',')
+ exclude = '-' if exclude == ""
+ extrapush = component_conf['extra-push'].sort().join(',')
+ extrapush = '-' if extrapush == ""
+
+ lines << "#{component_conf['master']} #{component_name} #{srchost} #{srcpath} #{extrapush} #{exclude}"
+end
+lines.join("\n")
+%>
projects / mirror / dsa-puppet.git / commitdiff