From: Peter Palfrader <peter@palfrader.org>
Date: Thu, 25 Apr 2013 12:15:37 +0000 (+0200)
Subject: Try to allow for individual pushes of static components at the cost of having a singl... 
X-Git-Url: https://git.adam-barratt.org.uk/?a=commitdiff_plain;h=b762782a06b2fff4d656866def1c5267835be269;p=mirror%2Fdsa-puppet.git

Try to allow for individual pushes of static components at the cost of having a single, big, consistent static tree
---

diff --git a/modules/roles/files/static-mirroring/static-components.conf b/modules/roles/files/static-mirroring/static-components.conf
index f028ff2d0..e4a1d6196 100644
--- a/modules/roles/files/static-mirroring/static-components.conf
+++ b/modules/roles/files/static-mirroring/static-components.conf
@@ -1,8 +1,8 @@
 # puppetd maintained
-# <service> <source host> <directory> <extra push hosts, comma separated>
+# <master> <service> <source host> <directory> <extra push hosts, comma separated>
 
-mozilla.debian.net	wagner.debian.org	/srv/home/groups/pkg-mozilla/htdocs
-planet.debian.org	senfl.debian.org	/srv/planet.debian.org/www
-www.debian.org		wolkenstein.debian.org	/srv/www.debian.org/www
-bits.debian.org		master.debian.org	/srv/bits-master.debian.org/htdocs
-backports.debian.org	ries.debian.org		/srv/backports.debian.org/htdocs		franck.debian.org
+bizet.debian.org	mozilla.debian.net	wagner.debian.org	/srv/home/groups/pkg-mozilla/htdocs
+bizet.debian.org	planet.debian.org	senfl.debian.org	/srv/planet.debian.org/www
+bizet.debian.org	www.debian.org		wolkenstein.debian.org	/srv/www.debian.org/www
+bizet.debian.org	bits.debian.org		master.debian.org	/srv/bits-master.debian.org/htdocs
+bizet.debian.org	backports.debian.org	ries.debian.org		/srv/backports.debian.org/htdocs		franck.debian.org
diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run
index 027f38468..fab3baa40 100755
--- a/modules/roles/files/static-mirroring/static-master-run
+++ b/modules/roles/files/static-mirroring/static-master-run
@@ -10,9 +10,6 @@ import tempfile
 import time
 
 base='/home/staticsync/static-master'
-subdirs = { 'master': 'master',       # where updates from off-site end up going, the source of everything we do here
-            'cur':    'current-push', # where clients rsync from during a mirror push
-            'live':   'current-live'} # what is currently on the mirrors, and what they rsync from when they come back from being down
 serialname = '.serial'
 
 clients = []
@@ -88,12 +85,12 @@ def stage2(pipes, status, command):
       log("%s >> %s"%(c, l))
     log("%s: returned %d"%(c, p.returncode))
 
-def callout(serial):
+def callout(component, serial):
   log("Calling clients...")
   pipes = {}
   status = {}
   for c in clients:
-    args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', "%d"%(serial,)]
+    args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)]
     p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     pipes[c] = p
     status[c] = 'in-progress'
@@ -117,17 +114,18 @@ def callout(serial):
 
 
 cleanup_dirs = []
-def run_mirror():
+def run_mirror(component):
   # setup
-  master = os.path.join(base, subdirs['master'])
-  cur = os.path.join(base, subdirs['cur'])
-  live = os.path.join(base, subdirs['live'])
-  tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_new);
-  tmpdir_old = tempfile.mkdtemp(prefix='live.old-', dir=base); cleanup_dirs.append(tmpdir_old);
+  basemaster = os.path.join(base, 'master')
+  componentdir = os.path.join(basemaster, component)
+  cur = componentdir + '-current-push'
+  live = componentdir + '-current-live'
+  tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
+  tmpdir_old = tempfile.mkdtemp(prefix='live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
   os.chmod(tmpdir_new, 0755)
 
   locks = []
-  for p in (master, live, tmpdir_new):
+  for p in (componentdir, live, tmpdir_new):
     if not os.path.exists(p): os.mkdir(p, 0755)
     fd = os.open(p, os.O_RDONLY)
     log("Acquiring lock for %s(%d)."%(p,fd))
@@ -135,7 +133,7 @@ def run_mirror():
     locks.append(fd)
   log("All locks acquired.")
 
-  serialfile = os.path.join(master, serialname)
+  serialfile = os.path.join(componentdir, serialname)
   try:
     with open(serialfile) as f: serial = int(f.read())
   except:
@@ -144,7 +142,7 @@ def run_mirror():
   log("Serial is %s."%(serial,))
 
   log("Populating %s."%(tmpdir_new,))
-  subprocess.check_call(['cp', '-al', os.path.join(master, '.'), tmpdir_new])
+  subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new])
 
   if os.path.exists(cur):
     log("Removing existing %s."%(cur,))
@@ -153,7 +151,7 @@ def run_mirror():
   log("Renaming %s to %s."%(tmpdir_new, cur))
   os.rename(tmpdir_new, cur)
 
-  proceed = callout(serial)
+  proceed = callout(component, serial)
 
   if proceed:
     log("Moving %s aside."%(live,))
@@ -174,9 +172,14 @@ def run_mirror():
   return ret
 
 
+if len(sys.argv) != 2:
+  print >> sys.stderr, "Usage: %s <component>"%(sys.argv[0],)
+  sys.exit(1)
+component = sys.argv[1]
+
 ok = False
 try:
-  ok = run_mirror()
+  ok = run_mirror(component)
 finally:
   for p in cleanup_dirs:
     if os.path.exists(p): shutil.rmtree(p)
diff --git a/modules/roles/files/static-mirroring/static-master-ssh-wrap b/modules/roles/files/static-mirroring/static-master-ssh-wrap
index 625d3d9b5..8387560b0 100755
--- a/modules/roles/files/static-mirroring/static-master-ssh-wrap
+++ b/modules/roles/files/static-mirroring/static-master-ssh-wrap
@@ -72,22 +72,26 @@ lock() {
 
 do_rsync() {
 	local remote_host="$1"; shift
-
 	local args="--server --sender -vlHtrze.iLsf --safe-links ."
-	if [ "$*" = "$args -new-/" ] || [ "$*" = "$args ./-new-/" ] ; then
-		local path="$BASEDIR/current-push"
-		info "serving $remote_host with $path"
-		rsync $args "$path/."
-	elif [ "$*" = "$args . -live-/" ] || [ "$*" = "$args . ./-live-/" ] ; then
-		local path="$BASEDIR/current-live"
-		info "host $remote_host wants $path, acquiring lock"
-		lock 200 "$path" 0
-		rsync $args "$path/."
-	else
-		info "NOT allowed for $remote_host: rsync $*"
-		echo >&2 "This rsync command ($@) not allowed."
-		exit 1
-	fi
+
+	for component in $(awk -v this_host="$(hostname -f)" '$1 == this_host {print $2}' $COMPONENTLIST); do
+	  if [ "$*" = "$args $component/-new-/" ] || [ "$*" = "$args ./$component/-new-/" ] ; then
+		  local path="$BASEDIR/$component-current-push"
+		  info "serving $remote_host with $path"
+		  rsync $args "$path/."
+		  return
+	  elif [ "$*" = "$args . $component/-live-/" ] || [ "$*" = "$args . ./$component/-live-/" ] ; then
+		  local path="$BASEDIR/$component-current-live"
+		  info "host $remote_host wants $path, acquiring lock"
+		  lock 200 "$path" 0
+		  rsync $args "$path/."
+		  return
+	  fi
+	done
+
+	info "NOT allowed for $remote_host: rsync $*"
+	echo >&2 "This rsync command ($@) not allowed."
+	exit 1
 }
 
 do_update_component() {
@@ -98,16 +102,16 @@ do_update_component() {
 	shift
 
 	hit="$(
-		awk -v component="$component" -v host="$remote_host" '
-		  $1 == component {
-			  if ($2 == host) {
-				  print $3
+		awk -v this_host="$(hostname -f)" component="$component" -v host="$remote_host" '
+		  $1 == this_host && $2 == component {
+			  if ($3 == host) {
+				  print $4
 				  exit
 			  }
-			  split($4,extra,",")
+			  split($5,extra,",")
 			  for (i in extra) {
 				  if (host == extra[i]) {
-					  printf "%s:%s\n", $2, $3
+					  printf "%s:%s\n", $3, $4
 					  exit
 				  }
 			  }
diff --git a/modules/roles/files/static-mirroring/static-master-update-component b/modules/roles/files/static-mirroring/static-master-update-component
index f2ea11653..82bae0d81 100755
--- a/modules/roles/files/static-mirroring/static-master-update-component
+++ b/modules/roles/files/static-mirroring/static-master-update-component
@@ -54,7 +54,7 @@ lock() {
   fi
 
   if ! flock "$locktype" "$fd"; then
-    echo >&2 "$0: Cannot acquire lock on $base (flock $locktype failed) - Very bad, we should have waited!"
+    echo >&2 "$0: Cannot acquire lock on $path (flock $locktype failed) - Very bad, we should have waited!"
     exit 1
   fi
 }
@@ -81,8 +81,8 @@ if [ "${component%/*}" != "$component" ] ; then
   exit 1
 fi
 
-srchost="$(awk -v component="$component" '$1 == component {print $2; exit}' "$componentlist")"
-srcdir="$(awk -v component="$component" '$1 == component {print $3; exit}' "$componentlist")"
+srchost="$(awk -v this_host="$(hostname -f)" component="$component" '$1 == this_host && $2 == component {print $3; exit}' "$componentlist")"
+srcdir="$(awk -v this_host="$(hostname -f)" component="$component" '$1 == this_host && $2 == component {print $4; exit}' "$componentlist")"
 if [ -z "$srchost" ] || [ -z "$srcdir" ]; then
   echo >&2 "$0: Invalid component: $component (not found in $componentlist)";
   exit 1
@@ -100,11 +100,10 @@ else
 fi
 
 echo "$0: Acquiring locks..."
-lock 200 "$base" 0
 lock 201 "$tgt" 1
 
-tmpdir_new="$(mktemp -d --tmpdir="$base" "${component}.new-XXXXXX")"
-tmpdir_old="$(mktemp -d --tmpdir="$base" "${component}.old-XXXXXX")"
+tmpdir_new="$(mktemp -d --tmpdir="$base" "${component}-updating.incoming-XXXXXX")"
+tmpdir_old="$(mktemp -d --tmpdir="$base" "${component}-updating.removing-XXXXXX")"
 trap "rm -rf '$tmpdir_new' '$tmpdir_old'" EXIT
 chmod 0755 "$tmpdir_new"
 
@@ -116,6 +115,7 @@ rsync --delete \
   -trz \
   --links --hard-links --safe-links \
   --link-dest="$tgt" \
+  --exclude='/.serial' \
   "$src/." "$tmpdir_new/."
 echo "$0: Done.  Committing."
 
@@ -124,18 +124,18 @@ if ! mv "$tmpdir_new" "$tgt"; then
   echo >&2 "$0: WARNING: could not move $tmpdir_new to $tgt.  Trying to recover"
   rm -rf "$tgt"
   mv "$tmpdir_old/old" "$tgt"
-  echo >&2 "$0: Rolled back to old tree maybe successfully."
+  echo >&2 "$0: Rolled back to old tree, maybe even successfully."
   exit 1
 fi
 
 rm -rf "$tmpdir_new" "$tmpdir_old"
 trap - EXIT
 
-date '+%s' > "$base/.serial"
+date '+%s' > "$tgt/.serial"
 unlock 201
-unlock 200
+unlock 202
 echo "$0: Triggering mirror runs..."
-exec static-master-run
+exec static-master-run "$component"
 
 # vim:set et:
 # vim:set ts=2:
diff --git a/modules/roles/files/static-mirroring/static-mirror-run b/modules/roles/files/static-mirroring/static-mirror-run
index 038700ddb..72f7aea61 100755
--- a/modules/roles/files/static-mirroring/static-mirror-run
+++ b/modules/roles/files/static-mirroring/static-mirror-run
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# initiate a mirror staged mirror update from sync-source.
+# initiate a staged mirror update from sync-source for a component.
 #
 # if we have a serial file and we got a serial on the command line, only sync if the serial is different
 
@@ -119,16 +119,18 @@ lock() {
 
 
 log_setup
-#log "called with $* and ${SSH_ORIGINAL_COMMAND:-no ssh original command options}."
 log "called with $*"
 lock
 
 if [ -e "${BASEDIR}${ACTIVE}" ] && [ "$(readlink "${BASEDIR}${ACTIVE}")" = "$ALPHA" ] ; then
 	staging="$BRAVO"
 	active="$ALPHA"
-else
+elif [ -e "${BASEDIR}${ACTIVE}" ] && [ "$(readlink "${BASEDIR}${ACTIVE}")" = "$BRAVO" ] ; then
 	staging="$ALPHA"
 	active="$BRAVO"
+else
+	echo >&5 "Invalid state of ${BASEDIR}${ACTIVE}."
+	exit 1
 fi
 log "active is $active; staging is $staging"
 
diff --git a/modules/roles/files/static-mirroring/static-mirror-ssh-wrap b/modules/roles/files/static-mirroring/static-mirror-ssh-wrap
index 240fefec8..b4c8447b8 100755
--- a/modules/roles/files/static-mirroring/static-mirror-ssh-wrap
+++ b/modules/roles/files/static-mirroring/static-mirror-ssh-wrap
@@ -36,7 +36,9 @@ COMPONENTLIST=/etc/static-components.conf
 usage() {
 	echo "local Usage: $0 <basedir> <host>"
 	echo "via ssh orig command:"
-	echo "                      mirror <serial>"
+	echo "                      mirror <component> <serial>"
+	echo "                      rsync ..."
+		do_rsync "$remote_host" "$@"
 }
 
 one_more_arg() {
@@ -59,10 +61,12 @@ do_mirror() {
 	local basedir="$1"; shift
 	local remote_host="$1"; shift
 	one_more_arg "$@"
+	local component="$1"; shift
+	one_more_arg "$@"
 	local serial="$1"; shift
 
-	info "Host $remote_host triggered a mirror run for serial $serial"
-	exec /usr/local/bin/static-mirror-run "$basedir" "$remote_host:-new-" "$serial"
+	info "Host $remote_host triggered a mirror run for $component, serial $serial"
+	exec /usr/local/bin/static-mirror-run "$basedir" "$remote_host:$component/-new-" "$serial"
 	echo >&2 "Exec failed"
 	croak "exec failed"
 }
@@ -75,7 +79,7 @@ do_rsync() {
 	allowed_rsyncs=()
 
 	if [ -e "$COMPONENTLIST" ]; then
-		for path in $(awk -v host="$(hostname -f)" '$2 == host {print $3}' $COMPONENTLIST); do
+		for path in $(awk -v host="$(hostname -f)" '$3 == host {print $4}' $COMPONENTLIST); do
 			allowed_rsyncs+=("--server --sender -lHtrze.iLsf --safe-links . $path/.")
 		done
 	fi
diff --git a/modules/roles/manifests/static_mirror.pp b/modules/roles/manifests/static_mirror.pp
index 8dd1bb850..c8d9335aa 100644
--- a/modules/roles/manifests/static_mirror.pp
+++ b/modules/roles/manifests/static_mirror.pp
@@ -24,7 +24,7 @@ class roles::static_mirror {
 	}
 
 	file { '/etc/cron.d/puppet-static-mirror':
-			content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; static-mirror-run --one-stage /srv/static.debian.org bizet.debian.org:-live- > /dev/null\n",
+			content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; for a in `awk '!/^ *(#|$)/ {printf "%s:%s/-live-\n", $1, $2}' /etc/static-components.conf`; do static-mirror-run --one-stage /srv/static.debian.org "$a" > /dev/null; done\n",
 	}
 
 	$vhost_listen = $::hostname ? {