- rautavaara.debian.org
- salieri.debian.org
- widor.debian.org
+ static_master:
+ - bizet.debian.org
+ static_mirror:
+ - klecker.debian.org
# reservedaddrs:
# ball.debian.org: "0.0.0.0/8 : 127.0.0.0/8 : 169.254.0.0/16 : 172.16.0.0/12 : 192.0.0.0/17 : 192.168.0.0/16 : 224.0.0.0/4 : 240.0.0.0/5 : 248.0.0.0/5"
--- /dev/null
+#!/usr/bin/python
+
+import fcntl
+import os
+import shutil
+import subprocess
+import string
+import tempfile
+import time
+
+base='/home/staticsync/static-master'
+subdirs = { 'master': 'master', # where updates from off-site end up going, the source of everything we do here
+ 'cur': 'current-push', # where clients rsync from during a mirror push
+ 'live': 'current-live'} # what is currently on the mirrors, and what they rsync from when they come back from being down
+serialname = '.serial'
+
+clients = []
+with open('/home/staticsync/etc/static-clients') as f:
+ for line in f:
+ clients.append(line.strip())
+
+def log(m):
+ t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
+ print t, m
+
+def stage1(pipes, status):
+ for c in clients:
+ p = pipes[c]
+ while 1:
+ line = p.stdout.readline()
+ if line == '':
+ status[c] = 'failed'
+ p.stdout.close()
+ p.stdin.close()
+ p.wait()
+ log("%s: failed with returncode %d"%(c,p.returncode))
+ break
+
+ line = line.strip()
+ log("%s >> %s"%(c, line))
+ if not line.startswith('[MSM]'): continue
+ kw = string.split(line, ' ', 2)[1]
+
+ if kw == 'ALREADY-CURRENT':
+ pipes[c].stdout.close()
+ pipes[c].stdin.close()
+ p.wait()
+ if p.returncode == 0:
+ log("%s: already current"%(c,))
+ status[c] = 'ok'
+ else:
+ log("%s: said ALREADY-CURRENT but returncode %d"%(c,p.returncode))
+ status[c] = 'failed'
+ break
+ elif kw == 'STAGE1-DONE':
+ log("%s: waiting"%(c,))
+ status[c] = 'waiting'
+ break
+ elif kw in ['STAGE1-START']:
+ pass
+ else:
+ log("%s: ignoring unknown line"%(c,))
+
+def count_statuses(status):
+ cnt = {}
+ for k in status:
+ v = status[k]
+ if v not in cnt: cnt[v] = 1
+ else: cnt[v] += 1
+ return cnt
+
+def stage2(pipes, status, command):
+ for c in clients:
+ if status[c] != 'waiting': continue
+ log("%s << %s"%(c, command))
+ pipes[c].stdin.write("%s\n"%(command,))
+
+ for c in clients:
+ if status[c] != 'waiting': continue
+ p = pipes[c]
+
+ (o, dummy) = p.communicate('')
+ for l in string.split(o, "\n"):
+ log("%s >> %s"%(c, l))
+ log("%s: returned %d"%(c, p.returncode))
+
+def callout(serial):
+ log("Calling clients...")
+ pipes = {}
+ status = {}
+ for c in clients:
+ args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', "%d"%(serial,)]
+ p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ pipes[c] = p
+ status[c] = 'in-progress'
+
+ log("Stage 1...")
+ stage1(pipes, status)
+ log("Stage 1 done.")
+ cnt = count_statuses(status)
+
+ if 'failed' in cnt > 0:
+ log("Some clients failed, aborting...")
+ stage2(pipes, status, 'abort')
+ return False
+ elif 'waiting' in cnt > 0:
+ log("Committing...")
+ stage2(pipes, status, 'go')
+ return True
+ else:
+ log("All clients up to date.")
+ return True
+
+
+cleanup_dirs = []
+def run_mirror():
+ # setup
+ master = os.path.join(base, subdirs['master'])
+ cur = os.path.join(base, subdirs['cur'])
+ live = os.path.join(base, subdirs['live'])
+ tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_new);
+ tmpdir_old = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_old);
+ os.chmod(tmpdir_new, 0755)
+
+ locks = []
+ for p in (master, live, tmpdir_new):
+ if not os.path.exists(p): os.mkdir(p, 0755)
+ fd = os.open(p, os.O_RDONLY)
+ log("Acquiring lock for %s(%d)."%(p,fd))
+ fcntl.flock(fd, fcntl.LOCK_EX)
+ locks.append(fd)
+ log("All locks acquired.")
+
+ serialfile = os.path.join(master, serialname)
+ try:
+ with open(serialfile) as f: serial = int(f.read())
+ except:
+ serial = int(time.time())
+ with open(serialfile, "w") as f: f.write("%d\n"%(serial,))
+ log("Serial is %s."%(serial,))
+
+ log("Populating %s."%(tmpdir_new,))
+ subprocess.check_call(['cp', '-al', os.path.join(master, '.'), tmpdir_new])
+
+ if os.path.exists(cur):
+ log("Removing existing %s."%(cur,))
+ shutil.rmtree(cur)
+
+ log("Renaming %s to %s."%(tmpdir_new, cur))
+ os.rename(tmpdir_new, cur)
+
+ proceed = callout(serial)
+
+ if proceed:
+ log("Moving %s aside."%(live,))
+ os.rename(live, os.path.join(tmpdir_old, 'old'))
+ log("Renaming %s to %s."%(cur, live))
+ os.rename(cur, live)
+ log("Cleaning up.")
+ shutil.rmtree(tmpdir_old)
+ log("Done.")
+ else:
+ log("Aborted.")
+
+
+try:
+ run_mirror()
+finally:
+ for p in cleanup_dirs:
+ if os.path.exists(p): shutil.rmtree(p)
+# vim:set et:
+# vim:set ts=2:
+# vim:set shiftwidth=2:
--- /dev/null
+#!/bin/bash
+
+# Copyright (c) 2009, 2010, 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+set -e
+set -u
+
+MYLOGNAME="`basename "$0"`[$$]"
+BASEDIR="/home/staticsync/static-master"
+
+usage() {
+ echo "local Usage: $0 <host>"
+ echo "via ssh orig command:"
+ echo " rsync <stuff>"
+ echo " static-master-update-component <component>"
+}
+
+one_more_arg() {
+ if [ "$#" -lt 1 ]; then
+ usage >&2
+ exit 1
+ fi
+}
+
+info() {
+ logger -p daemon.info -t "$MYLOGNAME" "$1"
+}
+
+croak() {
+ logger -s -p daemon.warn -t "$MYLOGNAME" "$1"
+ exit 1
+}
+
+lock() {
+ local fd="$1"; shift
+ local path="$1"; shift
+ local exclusive="$1"; shift
+
+ eval "exec $fd< '$path'"
+
+ if [ "$exclusive" -gt 0 ]; then
+ locktype="-e"
+ else
+ locktype="-s"
+ fi
+
+ if ! flock "$locktype" "$fd"; then
+ echo >&2 "$0: Cannot acquire lock on $base (flock $locktype failed) - Very bad, we should have waited!"
+ exit 1
+ fi
+}
+
+serve_dir() {
+ local remote_host="$1"; shift
+ local path="$1"; shift
+
+ local sender='rsync --server --sender -vlogDtprze.iLsf . '
+
+ if [ -e "$path" ]; then
+ info "serving $remote_host with $path"
+ $sender "$path/"
+ else
+ info "$remote_host wants non-existing $path"
+ echo >&2 "$path does not exist."
+ exit 1
+ fi
+}
+
+do_rsync() {
+ local remote_host="$1"; shift
+
+ if [ "$*" = "--server --sender -vlogDtprze.iLsf . -new-/" ] ; then
+ serve_dir "$remote_host" "$BASEDIR/current-push"
+ elif [ "$*" = "--server --sender -vlogDtprze.iLsf . -live-/" ] ; then
+ local p="$BASEDIR/current-live"
+ info "host $remote_host wants $p, acquiring lock"
+ lock 200 "$p" 0
+ serve_dir "$remote_host" "$p"
+ else
+ info "NOT allowed for $remote_host: rsync $*"
+ echo >&2 "This rsync command ($@) not allowed."
+ exit 1
+ fi
+}
+
+do_update_component() {
+ local remote_host="$1"; shift
+
+ one_more_arg "$@"
+ component="$1"
+ shift
+
+ #if [ "$component" = "www.torproject.org" ] && [ "$remote_host" = "vescum.torproject.org" ]; then
+ # exec static-master-update-component "$component"
+ # echo >&2 "Exec failed"
+ # croak "exec failed"
+ #else
+ info "Not whitelisted: $remote_host update $component"
+ echo >&2 "Not whitelisted: $remote_host update $component"
+ exit 1
+ #fi
+}
+
+
+if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
+ usage
+ exit 0
+fi
+
+one_more_arg "$@"
+remote_host="$1"
+shift
+
+
+# check/parse remote command line
+if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then
+ croak "Did not find SSH_ORIGINAL_COMMAND"
+fi
+set "dummy" ${SSH_ORIGINAL_COMMAND}
+shift
+
+info "host $remote_host called with $*"
+
+one_more_arg "$@"
+action="$1"
+shift
+
+case "$action" in
+ rsync)
+ do_rsync "$remote_host" "$@"
+ ;;
+ static-master-update-component)
+ do_update_component "$remote_host" "$@"
+ ;;
+ *)
+ croak "Invalid operation '$action'"
+ ;;
+esac
--- /dev/null
+#!/bin/bash
+
+# Updates one component (i.e. subdirectory) in static-master/master
+
+# acquires a shared lock on the base directory (so that we know no updates are
+# outgoing, as those acquire an exclusive one). Also acquired an exclusive lock
+# on the component directory in question.
+#
+# The config file is a list of component source-directory pairs.
+
+# Copyright (c) 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+componentlist=/home/staticsync/etc/static-components
+base=/home/staticsync/static-master/master
+
+set -e
+set -u
+
+
+lock() {
+ local fd="$1"; shift
+ local path="$1"; shift
+ local exclusive="$1"; shift
+
+ eval "exec $fd< '$path'"
+
+ if [ "$exclusive" -gt 0 ]; then
+ locktype="-e"
+ else
+ locktype="-s"
+ fi
+
+ if ! flock "$locktype" "$fd"; then
+ echo >&2 "$0: Cannot acquire lock on $base (flock $locktype failed) - Very bad, we should have waited!"
+ exit 1
+ fi
+}
+
+unlock() {
+ local fd="$1"; shift
+
+ if ! flock -o "$fd"; then
+ echo >&2 "$0: Cannot release lock on fd $fd - This should not have happened!"
+ exit 1
+ fi
+ eval "exec $fd<&-"
+}
+
+if [ "$#" != 1 ]; then
+ echo >&2 "Usage: $0 <component>"
+ exit 1
+fi
+
+component="$1"
+
+if [ "${component%/*}" != "$component" ] ; then
+ echo >&2 "$0: Invalid component: $component";
+ exit 1
+fi
+
+src="$(awk -v component="$component" '$1 == component {print $2; exit}' "$componentlist")"
+if [ -z "$src" ]; then
+ echo >&2 "$0: Invalid component: $component (not found in $componentlist)";
+ exit 1
+fi
+tgt="$base/$component"
+if ! [ -d "$tgt" ]; then
+ echo >&2 "$0: Invalid component: $component ($tgt does not exist)";
+ exit 1
+fi
+
+echo "$0: Acquiring locks..."
+lock 200 "$base" 0
+lock 201 "$tgt" 1
+
+tmpdir_new="$(mktemp -d --tmpdir="$base" "${component}.new-XXXXXX")"
+tmpdir_old="$(mktemp -d --tmpdir="$base" "${component}.old-XXXXXX")"
+trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT
+chmod 0755 "$tmpdir_new"
+
+lock 202 "$tmpdir_new" 1
+echo "$0: Got them."
+
+echo "$0: Updating master copy of $component..."
+rsync --delete \
+ -tr \
+ --link-dest="$tgt" \
+ "$src/." "$tmpdir_new/."
+echo "$0: Done. Committing."
+
+mv "$tgt" "$tmpdir_old/old"
+if ! mv "$tmpdir_new" "$tgt"; then
+ echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt. Trying to recover"
+ rm -rf "$tgt"
+ mv "$tmpdir_old/old" "$tgt"
+ echo >&2 "$0: Rolled back to old tree maybe successfully."
+ exit 1
+fi
+
+rm -rf "$tmpdir_new" "$tmpdir_old"
+trap - EXIT
+
+date '+%s' > "$base/.serial"
+unlock 201
+unlock 200
+echo "$0: Triggering mirror runs..."
+exec static-master-run
+
+# vim:set et:
+# vim:set ts=2:
+# vim:set shiftwidth=2:
--- /dev/null
+#!/bin/bash
+
+# initiate a mirror staged mirror update from sync-source.
+#
+# if we have a serial file and we got a serial on the command line, only sync if the serial is different
+
+# Copyright (c) 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+set -e
+set -u
+
+NAME="$(basename "$0")"
+
+usage() {
+ echo "Usage: $0 [--one-stage] <basedir> <sync-source> [<serial>]"
+}
+
+if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then usage; exit 0; fi
+
+one_stage=0
+while :; do
+ case "${1:-}" in
+ --)
+ shift
+ break;
+ ;;
+ --one-stage)
+ shift
+ one_stage=1
+ ;;
+ -*) usage >&2
+ exit 1
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+BASEDIR=${1:-}; shift
+SYNC_SOURCE=${1:-}; shift
+SYNC_SERIAL=${1:-}; shift || true
+if [ -z "$BASEDIR" ]; then usage >&2; exit 1; fi
+if [ -z "$SYNC_SOURCE" ]; then usage >&2; exit 1; fi
+
+RSYNC="rsync"
+RSYNC_BASE_OPTIONS="-avz --delete"
+RSYNC_SSH_OPTIONS="ssh -o AddressFamily=inet -o BatchMode=yes"
+
+LOGDIR="$HOME/logs"
+LOGFILE="$LOGDIR/$NAME-run.log"
+
+
+ALPHA="tree-a"
+BRAVO="tree-b"
+ACTIVE="cur"
+
+CNF_FILE="$HOME/etc/$NAME.conf"
+! [ -e "$CNF_FILE" ] || . "$CNF_FILE"
+
+SOURCE="${SYNC_SOURCE}/"
+BASEDIR="${BASEDIR}/"
+
+###############################################
+
+# point stdout and stderr to the logfile if it's not a tty.
+# save stdout to fd5 for communications with the master
+
+log_setup() {
+ mkdir -p "$LOGDIR"
+ if ! [ -t 1 ]; then
+ # move current stdout to fd5 and reopen to logfile
+ exec 5>&1-
+ exec 1>> "$LOGFILE"
+ else
+ # duplicate stdout to fd5
+ exec 5>&1
+ fi
+ if ! [ -t 2 ]; then
+ exec 2>> "$LOGFILE"
+ fi
+}
+
+log() {
+ echo "[$(date)][$NAME][$$] $1"
+}
+
+lock() {
+ exec 200< "$BASEDIR"
+ if ! flock -e 200; then
+ log "Cannot acquire lock."
+ echo >&5 "[MSM] LOCK-ERROR"
+ exit 1
+ fi
+ log "Got the lock."
+}
+
+###############################################
+
+
+log_setup
+#log "called with $* and ${SSH_ORIGINAL_COMMAND:-no ssh original command options}."
+log "called with $*"
+lock
+
+if [ -e "${BASEDIR}${ACTIVE}" ] && [ "$(readlink "${BASEDIR}${ACTIVE}")" = "$ALPHA" ] ; then
+ staging="$BRAVO"
+ active="$ALPHA"
+else
+ staging="$ALPHA"
+ active="$BRAVO"
+fi
+log "active is $active; staging is $staging"
+
+rsync_source="${SOURCE}"
+rsync_curactive="${BASEDIR}${active}/"
+rsync_target="${BASEDIR}${staging}/"
+
+if [ -e "$rsync_curactive/.serial" ] && [ -n "$SYNC_SERIAL" ] && [ "$(cat $rsync_curactive/.serial)" = "$SYNC_SERIAL" ]; then
+ log "active is already at serial $SYNC_SERIAL. No action required."
+ echo >&5 "[MSM] ALREADY-CURRENT"
+ exit 0
+fi
+
+echo >&5 "[MSM] STAGE1-START"
+log "Running $RSYNC $RSYNC_BASE_OPTIONS -e $RSYNC_SSH_OPTIONS --link-dest $rsync_curactive $rsync_source $rsync_target"
+$RSYNC $RSYNC_BASE_OPTIONS -e "$RSYNC_SSH_OPTIONS" --link-dest "$rsync_curactive" "$rsync_source" "$rsync_target"
+log "rsync done."
+echo >&5 "[MSM] STAGE1-DONE"
+if [ "$one_stage" -gt 0 ]; then
+ action="go"
+else
+ read action
+fi
+
+case "$action" in
+ go)
+ ln --symbolic --force --no-target-directory "$staging" "${BASEDIR}$ACTIVE"
+ rm -rf "$rsync_curactive"
+ echo >&5 "[MSM] STAGE2-DONE"
+ log "stage2 done"
+ ;;
+ abort)
+ echo >&5 "[MSM] STAGE2-ABORT"
+ log "stage2 abort"
+ ;;
+ *)
+ echo >&5 "[MSM] STAGE2-UNKNOWN-ACTION $action"
+ log "stage2 unknown action $action"
+ exit 1
+ ;;
+esac
+
+savelog "$LOGFILE"
--- /dev/null
+#!/bin/bash
+
+# Copyright (c) 2009, 2010, 2012 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+set -e
+set -u
+
+MYLOGNAME="`basename "$0"`[$$]"
+
+usage() {
+ echo "local Usage: $0 <basedir> <host>"
+ echo "via ssh orig command:"
+ echo " mirror <serial>"
+}
+
+one_more_arg() {
+ if [ "$#" -lt 1 ]; then
+ usage >&2
+ exit 1
+ fi
+}
+
+info() {
+ logger -p daemon.info -t "$MYLOGNAME" "$1"
+}
+
+croak() {
+ logger -s -p daemon.warn -t "$MYLOGNAME" "$1"
+ exit 1
+}
+
+do_mirror() {
+ local basedir="$1"; shift
+ local remote_host="$1"; shift
+ one_more_arg "$@"
+ local serial="$1"; shift
+
+ info "Host $remote_host triggered a mirror run for serial $serial"
+ exec /usr/local/bin/static-mirror-run "$basedir" "$remote_host:-new-" "$serial"
+ echo >&2 "Exec failed"
+ croak "exec failed"
+}
+
+do_rsync() {
+ local remote_host="$1"
+ shift
+
+ local allowed_rsyncs
+ allowed_rsyncs=()
+
+ #case "`hostname`" in
+ # vescum)
+ # allowed_rsyncs=(
+ # '^--server --sender -tre\.iLsf \. /srv/www-master\.torproject\.org/htdocs/\.$'
+ # )
+ # ;;
+ # *)
+ #esac
+ for cmd_idx in ${!allowed_rsyncs[*]}; do
+ allowed="${allowed_rsyncs[$cmd_idx]}"
+ if [[ "$*" =~ $allowed ]]; then # do !not! quote regex
+ info "Running for host $remote_host: rsync $*"
+ exec rsync "$@"
+ echo >&2 "Exec failed"
+ exit 1
+ fi
+ done
+
+ info "NOT allowed for $remote_host: rsync $*"
+ echo >&2 "This rsync command ($*) not allowed."
+ exit 1
+}
+
+
+if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
+ usage
+ exit 0
+fi
+
+one_more_arg "$@"
+basedir="$1"
+shift
+
+one_more_arg "$@"
+remote_host="$1"
+shift
+
+
+# check/parse remote command line
+if [ -z "${SSH_ORIGINAL_COMMAND:-}" ] ; then
+ croak "Did not find SSH_ORIGINAL_COMMAND"
+fi
+set "dummy" ${SSH_ORIGINAL_COMMAND}
+shift
+
+one_more_arg "$@"
+action="$1"
+shift
+
+case "$action" in
+ mirror)
+ do_mirror "$basedir" "$remote_host" "$@"
+ ;;
+ rsync)
+ do_rsync "$remote_host" "$@"
+ ;;
+ *)
+ croak "Invalid operation '$action'"
+ ;;
+esac
if getfromhash($site::nodeinfo, 'apache2_ftp-upcoming_mirror') {
include roles::ftp-upcoming_mirror
}
+
+ if getfromhash($site::nodeinfo, 'static_master') {
+ include roles::static_master
+ }
+
+ if getfromhash($site::nodeinfo, 'static_mirror') {
+ include roles::static_mirror
+ }
}
--- /dev/null
+class roles::static_master inherits roles::mirror_base {
+ file {
+ '/etc/ssh/userkeys/staticsync':
+ content => template('roles/static-master-authorized_keys.erb'),
+ ;
+ '/usr/local/bin/static-master-run':
+ source => "puppet:///modules/roles/static-mirroring/static-master-run",
+ mode => 555,
+ ;
+ '/usr/local/bin/static-master-ssh-wrap':
+ source => "puppet:///modules/roles/static-mirroring/static-master-ssh-wrap",
+ mode => 555,
+ ;
+ '/usr/local/bin/static-master-update-component':
+ source => "puppet:///modules/roles/static-mirroring/static-master-update-component",
+ mode => 555,
+ ;
+ }
+}
+# vim:set et:
+# vim:set sts=4 ts=4:
+# vim:set shiftwidth=4:
--- /dev/null
+class roles::static_mirror inherits roles::mirror_base {
+ file {
+ '/etc/ssh/userkeys/staticsync':
+ content => template('roles/static-mirror-authorized_keys.erb'),
+ ;
+ '/usr/local/bin/static-mirror-run':
+ source => "puppet:///modules/roles/static-mirroring/static-mirror-run",
+ mode => 555,
+ ;
+ '/usr/local/bin/static-mirror-ssh-wrap':
+ source => "puppet:///modules/roles/static-mirroring/static-mirror-ssh-wrap",
+ mode => 555,
+ ;
+ }
+
+ file {
+ "/etc/cron.d/puppet-static-mirror":
+ content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; static-mirror-run --one-stage /srv/static.debian.org bizet.debian.org:-live- > /dev/null\n",
+ ;
+
+ #"/etc/apache2/sites-available/dist.torproject.org":
+ # source => "puppet:///modules/roles/static-mirroring/vhost/dist.torproject.org",
+ # require => Package["apache2"],
+ # notify => Exec["reload-apache2"],
+ # ;
+ #"/etc/apache2/sites-available/www.torproject.org":
+ # source => "puppet:///modules/roles/static-mirroring/vhost/www.torproject.org",
+ # require => Package["apache2"],
+ # notify => Exec["reload-apache2"],
+ # ;
+ }
+
+ #apache2::activate_apache_site {
+ # "10-dist.torproject.org":
+ # site => "dist.torproject.org",
+ # require => File['/etc/ssl/certs/apache-wildcard.torproject.org.pem'];
+ # "10-www.torproject.org":
+ # site => "www.torproject.org",
+ # require => File['/etc/ssl/certs/apache-wildcard.torproject.org.pem'];
+ #}
+}
+# vim:set et:
+# vim:set sts=4 ts=4:
+# vim:set shiftwidth=4:
--- /dev/null
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+##
+
+<%=
+def getstaticsynckey(host)
+ key = nil
+ begin
+ facts = YAML.load(File.open("/var/lib/puppet/yaml/facts/#{host}.yaml").read)
+ return facts.values['staticsync_key']
+ rescue Exception => e
+ end
+ return key
+end
+
+mirrors = []
+scope.lookupvar('site::localinfo').keys.sort.each do |node|
+ if scope.lookupvar('site::localinfo')[node]['static_mirror']
+ key = getstaticsynckey(node)
+ mirrors << { 'node' => node, 'addr' => scope.lookupvar('site::allnodeinfo')[node]['ipHostNumber'], 'key' => key}
+ end
+end
+
+
+lines = []
+for m in mirrors:
+ lines << '# ' + m['node']
+ if m['key'].nil?
+ lines << "# no key for node"
+ else
+ lines << "command=\"/usr/local/bin/static-master-ssh-wrap #{m['node']}\"," +
+ 'no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-user-rc,' +
+ 'from="' + m['addr'].join(',') + '" ' +
+ m['key']
+ end
+end
+
+lines.join("\n")
+# vim:set et:
+# vim:set sts=4 ts=4:
+# vim:set shiftwidth=4:
+%>
--- /dev/null
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+##
+
+<%=
+def getstaticsynckey(host)
+ key = nil
+ begin
+ facts = YAML.load(File.open("/var/lib/puppet/yaml/facts/#{host}.yaml").read)
+ return facts.values['staticsync_key']
+ rescue Exception => e
+ end
+ return key
+end
+
+masters = []
+scope.lookupvar('site::localinfo').keys.sort.each do |node|
+ if scope.lookupvar('site::localinfo')[node]['static_master']
+ key = getstaticsynckey(node)
+ masters << { 'node' => node, 'addr' => scope.lookupvar('site::allnodeinfo')[node]['ipHostNumber'], 'key' => key}
+ end
+end
+
+
+lines = []
+for m in masters:
+ lines << '# ' + m['node']
+ if m['key'].nil?
+ lines << "# no key for node"
+ else
+ lines << "command=\"/usr/local/bin/static-mirror-ssh-wrap /srv/static.debian.org #{m['node']}\"," +
+ 'no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-user-rc,' +
+ 'from="' + m['addr'].join(',') + '" ' +
+ m['key']
+ end
+end
+
+lines.join("\n")
+# vim:set et:
+# vim:set sts=4 ts=4:
+# vim:set shiftwidth=4:
+%>
projects / mirror / dsa-puppet.git / commitdiff