import tempfile
import time
-base='/home/staticsync/static-master'
-subdirs = { 'master': 'master', # where updates from off-site end up going, the source of everything we do here
- 'cur': 'current-push', # where clients rsync from during a mirror push
- 'live': 'current-live'} # what is currently on the mirrors, and what they rsync from when they come back from being down
+base="/srv/static.debian.org"
serialname = '.serial'
+had_warnings = False
clients = []
with open('/etc/static-clients.conf') as f:
log("%s >> %s"%(c, l))
log("%s: returned %d"%(c, p.returncode))
-def callout(serial):
+def callout(component, serial):
log("Calling clients...")
pipes = {}
status = {}
for c in clients:
- args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', "%d"%(serial,)]
+ args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)]
p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
pipes[c] = p
status[c] = 'in-progress'
log("Stage 1 done.")
cnt = count_statuses(status)
- if 'failed' in cnt > 0:
- log("Some clients failed, aborting...")
+ if 'failed' in cnt and cnt['failed'] >= 2:
+ log("%d clients failed, aborting..."%(cnt['failed'],))
stage2(pipes, status, 'abort')
return False
- elif 'waiting' in cnt > 0:
+
+ failedmirrorsfile = os.path.join(base, 'master', component + "-failedmirrors")
+ if 'failed' in cnt:
+ log("WARNING: %d clients failed! Continuing anyway!"%(cnt['failed'],))
+ global had_warnings
+ had_warnings = True
+ f = open(failedmirrorsfile, "w")
+ for c in status:
+ if status[c] == 'failed': f.write(c+"\n")
+ f.close()
+ else:
+ if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
+
+ if 'waiting' in cnt:
log("Committing...")
stage2(pipes, status, 'go')
return True
cleanup_dirs = []
-def run_mirror():
+def run_mirror(component):
# setup
- master = os.path.join(base, subdirs['master'])
- cur = os.path.join(base, subdirs['cur'])
- live = os.path.join(base, subdirs['live'])
- tmpdir_new = tempfile.mkdtemp(prefix='live.new-', dir=base); cleanup_dirs.append(tmpdir_new);
- tmpdir_old = tempfile.mkdtemp(prefix='live.old-', dir=base); cleanup_dirs.append(tmpdir_old);
+ basemaster = os.path.join(base, 'master')
+ componentdir = os.path.join(basemaster, component)
+ cur = componentdir + '-current-push'
+ live = componentdir + '-current-live'
+ tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
+ tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
os.chmod(tmpdir_new, 0755)
locks = []
- for p in (master, live, tmpdir_new):
+ for p in (componentdir, live, tmpdir_new):
if not os.path.exists(p): os.mkdir(p, 0755)
fd = os.open(p, os.O_RDONLY)
log("Acquiring lock for %s(%d)."%(p,fd))
locks.append(fd)
log("All locks acquired.")
- serialfile = os.path.join(master, serialname)
+ serialfile = os.path.join(componentdir, serialname)
try:
with open(serialfile) as f: serial = int(f.read())
except:
log("Serial is %s."%(serial,))
log("Populating %s."%(tmpdir_new,))
- subprocess.check_call(['cp', '-al', os.path.join(master, '.'), tmpdir_new])
+ subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new])
if os.path.exists(cur):
log("Removing existing %s."%(cur,))
log("Renaming %s to %s."%(tmpdir_new, cur))
os.rename(tmpdir_new, cur)
- proceed = callout(serial)
+ proceed = callout(component, serial)
if proceed:
log("Moving %s aside."%(live,))
os.rename(cur, live)
log("Cleaning up.")
shutil.rmtree(tmpdir_old)
- log("Done.")
+ if had_warnings: log("Done, with warnings.")
+ else: log("Done.")
ret = True
else:
log("Aborted.")
ret = False
for fd in locks:
- fd.close()
+ os.close(fd)
return ret
+if len(sys.argv) != 2:
+ print >> sys.stderr, "Usage: %s <component>"%(sys.argv[0],)
+ sys.exit(1)
+component = sys.argv[1]
+
ok = False
try:
- ok = run_mirror()
+ ok = run_mirror(component)
finally:
for p in cleanup_dirs:
if os.path.exists(p): shutil.rmtree(p)