# puppetd maintained
-# <service> <source host> <directory> <extra push hosts, comma separated>
+# <master> <service> <source host> <directory> <extra push hosts, comma separated>
-mozilla.debian.net wagner.debian.org /srv/home/groups/pkg-mozilla/htdocs
-planet.debian.org senfl.debian.org /srv/planet.debian.org/www
-www.debian.org wolkenstein.debian.org /srv/www.debian.org/www
-bits.debian.org master.debian.org /srv/bits-master.debian.org/htdocs
-backports.debian.org ries.debian.org /srv/backports.debian.org/htdocs franck.debian.org
+bizet.debian.org mozilla.debian.net wagner.debian.org /srv/home/groups/pkg-mozilla/htdocs
+bizet.debian.org planet.debian.org senfl.debian.org /srv/planet.debian.org/www
+bizet.debian.org www.debian.org wolkenstein.debian.org /srv/www.debian.org/www
+bizet.debian.org bits.debian.org master.debian.org /srv/bits-master.debian.org/htdocs
+bizet.debian.org backports.debian.org ries.debian.org /srv/backports.debian.org/htdocs franck.debian.org
import time
base='/home/staticsync/static-master'
-subdirs = { 'master': 'master', # where updates from off-site end up going, the source of everything we do here
- 'cur': 'current-push', # where clients rsync from during a mirror push
- 'live': 'current-live'} # what is currently on the mirrors, and what they rsync from when they come back from being down
serialname = '.serial'
clients = []
log("%s >> %s"%(c, l))
log("%s: returned %d"%(c, p.returncode))
-def callout(serial):
+def callout(component, serial):
log("Calling clients...")
pipes = {}
status = {}
for c in clients:
- args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', "%d"%(serial,)]
+ args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)]
p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pipes[c] = p
status[c] = 'in-progress'
cleanup_dirs = []
-def run_mirror():
+def run_mirror(component):
# setup
- master = os.path.join(base, subdirs['master'])
- cur = os.path.join(base, subdirs['cur'])
- live = os.path.join(base, subdirs['live'])
- tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_new);
- tmpdir_old = tempfile.mkdtemp(prefix='live.old-', dir=base); cleanup_dirs.append(tmpdir_old);
+ basemaster = os.path.join(base, 'master')
+ componentdir = os.path.join(basemaster, component)
+ cur = componentdir + '-current-push'
+ live = componentdir + '-current-live'
+ tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
+ tmpdir_old = tempfile.mkdtemp(prefix='live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
os.chmod(tmpdir_new, 0755)
locks = []
- for p in (master, live, tmpdir_new):
+ for p in (componentdir, live, tmpdir_new):
if not os.path.exists(p): os.mkdir(p, 0755)
fd = os.open(p, os.O_RDONLY)
log("Acquiring lock for %s(%d)."%(p,fd))
locks.append(fd)
log("All locks acquired.")
- serialfile = os.path.join(master, serialname)
+ serialfile = os.path.join(componentdir, serialname)
try:
with open(serialfile) as f: serial = int(f.read())
except:
log("Serial is %s."%(serial,))
log("Populating %s."%(tmpdir_new,))
- subprocess.check_call(['cp', '-al', os.path.join(master, '.'), tmpdir_new])
+ subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new])
if os.path.exists(cur):
log("Removing existing %s."%(cur,))
log("Renaming %s to %s."%(tmpdir_new, cur))
os.rename(tmpdir_new, cur)
- proceed = callout(serial)
+ proceed = callout(component, serial)
if proceed:
log("Moving %s aside."%(live,))
return ret
+if len(sys.argv) != 2:
+ print >> sys.stderr, "Usage: %s <component>"%(sys.argv[0],)
+ sys.exit(1)
+component = sys.argv[1]
+
ok = False
try:
- ok = run_mirror()
+ ok = run_mirror(component)
finally:
for p in cleanup_dirs:
if os.path.exists(p): shutil.rmtree(p)
do_rsync() {
local remote_host="$1"; shift
-
local args="--server --sender -vlHtrze.iLsf --safe-links ."
- if [ "$*" = "$args -new-/" ] || [ "$*" = "$args ./-new-/" ] ; then
- local path="$BASEDIR/current-push"
- info "serving $remote_host with $path"
- rsync $args "$path/."
- elif [ "$*" = "$args . -live-/" ] || [ "$*" = "$args . ./-live-/" ] ; then
- local path="$BASEDIR/current-live"
- info "host $remote_host wants $path, acquiring lock"
- lock 200 "$path" 0
- rsync $args "$path/."
- else
- info "NOT allowed for $remote_host: rsync $*"
- echo >&2 "This rsync command ($@) not allowed."
- exit 1
- fi
+
+ for component in $(awk -v this_host="$(hostname -f)" '$1 == this_host {print $2}' $COMPONENTLIST); do
+ if [ "$*" = "$args $component/-new-/" ] || [ "$*" = "$args ./$component/-new-/" ] ; then
+ local path="$BASEDIR/$component-current-push"
+ info "serving $remote_host with $path"
+ rsync $args "$path/."
+ return
+ elif [ "$*" = "$args . $component/-live-/" ] || [ "$*" = "$args . ./$component/-live-/" ] ; then
+ local path="$BASEDIR/$component-current-live"
+ info "host $remote_host wants $path, acquiring lock"
+ lock 200 "$path" 0
+ rsync $args "$path/."
+ return
+ fi
+ done
+
+ info "NOT allowed for $remote_host: rsync $*"
+ echo >&2 "This rsync command ($@) not allowed."
+ exit 1
}
do_update_component() {
shift
hit="$(
- awk -v component="$component" -v host="$remote_host" '
- $1 == component {
- if ($2 == host) {
- print $3
+ awk -v this_host="$(hostname -f)" component="$component" -v host="$remote_host" '
+ $1 == this_host && $2 == component {
+ if ($3 == host) {
+ print $4
exit
}
- split($4,extra,",")
+ split($5,extra,",")
for (i in extra) {
if (host == extra[i]) {
- printf "%s:%s\n", $2, $3
+ printf "%s:%s\n", $3, $4
exit
}
}
fi
if ! flock "$locktype" "$fd"; then
- echo >&2 "$0: Cannot acquire lock on $base (flock $locktype failed) - Very bad, we should have waited!"
+ echo >&2 "$0: Cannot acquire lock on $path (flock $locktype failed) - Very bad, we should have waited!"
exit 1
fi
}
exit 1
fi
-srchost="$(awk -v component="$component" '$1 == component {print $2; exit}' "$componentlist")"
-srcdir="$(awk -v component="$component" '$1 == component {print $3; exit}' "$componentlist")"
+srchost="$(awk -v this_host="$(hostname -f)" component="$component" '$1 == this_host && $2 == component {print $3; exit}' "$componentlist")"
+srcdir="$(awk -v this_host="$(hostname -f)" component="$component" '$1 == this_host && $2 == component {print $4; exit}' "$componentlist")"
if [ -z "$srchost" ] || [ -z "$srcdir" ]; then
echo >&2 "$0: Invalid component: $component (not found in $componentlist)";
exit 1
fi
echo "$0: Acquiring locks..."
-lock 200 "$base" 0
lock 201 "$tgt" 1
-tmpdir_new="$(mktemp -d --tmpdir="$base" "${component}.new-XXXXXX")"
-tmpdir_old="$(mktemp -d --tmpdir="$base" "${component}.old-XXXXXX")"
+tmpdir_new="$(mktemp -d --tmpdir="$base" "${component}-updating.incoming-XXXXXX")"
+tmpdir_old="$(mktemp -d --tmpdir="$base" "${component}-updating.removing-XXXXXX")"
trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT
chmod 0755 "$tmpdir_new"
-trz \
--links --hard-links --safe-links \
--link-dest="$tgt" \
+ --exclude='/.serial' \
"$src/." "$tmpdir_new/."
echo "$0: Done. Committing."
echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt. Trying to recover"
rm -rf "$tgt"
mv "$tmpdir_old/old" "$tgt"
- echo >&2 "$0: Rolled back to old tree maybe successfully."
+ echo >&2 "$0: Rolled back to old tree, maybe even successfully."
exit 1
fi
rm -rf "$tmpdir_new" "$tmpdir_old"
trap - EXIT
-date '+%s' > "$base/.serial"
+date '+%s' > "$tgt/.serial"
unlock 201
-unlock 200
+unlock 202
echo "$0: Triggering mirror runs..."
-exec static-master-run
+exec static-master-run "$component"
# vim:set et:
# vim:set ts=2:
#!/bin/bash
-# initiate a mirror staged mirror update from sync-source.
+# initiate a staged mirror update from sync-source for a component.
#
# if we have a serial file and we got a serial on the command line, only sync if the serial is different
log_setup
-#log "called with $* and ${SSH_ORIGINAL_COMMAND:-no ssh original command options}."
log "called with $*"
lock
if [ -e "${BASEDIR}${ACTIVE}" ] && [ "$(readlink "${BASEDIR}${ACTIVE}")" = "$ALPHA" ] ; then
staging="$BRAVO"
active="$ALPHA"
-else
+elif [ -e "${BASEDIR}${ACTIVE}" ] && [ "$(readlink "${BASEDIR}${ACTIVE}")" = "$BRAVO" ] ; then
staging="$ALPHA"
active="$BRAVO"
+else
+ echo >&5 "Invalid state of ${BASEDIR}${ACTIVE}."
+ exit 1
fi
log "active is $active; staging is $staging"
usage() {
echo "local Usage: $0 <basedir> <host>"
echo "via ssh orig command:"
- echo " mirror <serial>"
+ echo " mirror <component> <serial>"
+ echo " rsync ..."
+ do_rsync "$remote_host" "$@"
}
one_more_arg() {
local basedir="$1"; shift
local remote_host="$1"; shift
one_more_arg "$@"
+ local component="$1"; shift
+ one_more_arg "$@"
local serial="$1"; shift
- info "Host $remote_host triggered a mirror run for serial $serial"
- exec /usr/local/bin/static-mirror-run "$basedir" "$remote_host:-new-" "$serial"
+ info "Host $remote_host triggered a mirror run for $component, serial $serial"
+ exec /usr/local/bin/static-mirror-run "$basedir" "$remote_host:$component/-new-" "$serial"
echo >&2 "Exec failed"
croak "exec failed"
}
allowed_rsyncs=()
if [ -e "$COMPONENTLIST" ]; then
- for path in $(awk -v host="$(hostname -f)" '$2 == host {print $3}' $COMPONENTLIST); do
+ for path in $(awk -v host="$(hostname -f)" '$3 == host {print $4}' $COMPONENTLIST); do
allowed_rsyncs+=("--server --sender -lHtrze.iLsf --safe-links . $path/.")
done
fi
}
file { '/etc/cron.d/puppet-static-mirror':
- content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; static-mirror-run --one-stage /srv/static.debian.org bizet.debian.org:-live- > /dev/null\n",
+ content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; for a in `awk '!/^ *(#|$)/ {printf "%s:%s/-live-\n", $1, $2}' /etc/static-components.conf`; do static-mirror-run --one-stage /srv/static.debian.org "$a" > /dev/null; done\n",
}
$vhost_listen = $::hostname ? {
projects / mirror / dsa-puppet.git / commitdiff