X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=modules%2Froles%2Ffiles%2Fstatic-mirroring%2Fstatic-master-run;h=95c9355a5ffc6172f550f8489a09c1ebbae47fd7;hb=0aac917e02155dbb1b9f06d317f389feac08911d;hp=6ebe2be8129d90f0a0f7ef3a99341ac37b623655;hpb=cf710a2f5fa536f601433238b3e6de63ccaa288b;p=mirror%2Fdsa-puppet.git

diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run
index 6ebe2be81..95c9355a5 100755
--- a/modules/roles/files/static-mirroring/static-master-run
+++ b/modules/roles/files/static-mirroring/static-master-run
@@ -9,22 +9,36 @@ import sys
 import tempfile
 import time
 
-base='/home/staticsync/static-master'
 serialname = '.serial'
+had_warnings = False
 
-clients = []
+conffile = '/etc/staticsync.conf'
+config={}
+
+with open(conffile) as f:
+  for line in f:
+    line = line.rstrip()
+    if not line or line.startswith("#"): continue
+    (name, value) = line.split("=")
+    config[name] = value
+
+for key in ('base',):
+  if not key in config:
+    raise Exception("Configuration element '%s' not found in config file %s"%(key, conffile))
+
+allclients = set()
 with open('/etc/static-clients.conf') as f:
   for line in f:
     line = line.strip()
     if line == "": continue
     if line.startswith('#'): continue
-    clients.append(line)
+    allclients.add(line)
 
 def log(m):
   t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
   print t, m
 
-def stage1(pipes, status):
+def stage1(pipes, status, clients):
   for c in clients:
     p = pipes[c]
     while 1:
@@ -70,7 +84,7 @@ def count_statuses(status):
     else: cnt[v] += 1
   return cnt
 
-def stage2(pipes, status, command):
+def stage2(pipes, status, command, clients):
   for c in clients:
     if status[c] != 'waiting': continue
     log("%s << %s"%(c, command))
@@ -85,7 +99,7 @@ def stage2(pipes, status, command):
       log("%s >> %s"%(c, l))
     log("%s: returned %d"%(c, p.returncode))
 
-def callout(component, serial):
+def callout(component, serial, clients):
   log("Calling clients...")
   pipes = {}
   status = {}
@@ -96,27 +110,62 @@ def callout(component, serial):
     status[c] = 'in-progress'
 
   log("Stage 1...")
-  stage1(pipes, status)
+  stage1(pipes, status, clients)
   log("Stage 1 done.")
   cnt = count_statuses(status)
 
-  if 'failed' in cnt > 0:
-    log("Some clients failed, aborting...")
-    stage2(pipes, status, 'abort')
+  if 'failed' in cnt and cnt['failed'] >= 2:
+    log("%d clients failed, aborting..."%(cnt['failed'],))
+    stage2(pipes, status, 'abort', clients)
     return False
-  elif 'waiting' in cnt > 0:
+
+  failedmirrorsfile = os.path.join(config['base'], 'master', component + "-failedmirrors")
+  if 'failed' in cnt:
+    log("WARNING: %d clients failed!  Continuing anyway!"%(cnt['failed'],))
+    global had_warnings
+    had_warnings = True
+    f = open(failedmirrorsfile, "w")
+    for c in status:
+      if status[c] == 'failed': f.write(c+"\n")
+    f.close()
+  else:
+    if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
+
+  if 'waiting' in cnt:
     log("Committing...")
-    stage2(pipes, status, 'go')
+    stage2(pipes, status, 'go', clients)
     return True
   else:
     log("All clients up to date.")
     return True
 
+def load_component_info(component):
+  with open('/etc/static-components.conf') as f:
+    for line in f:
+      if line.startswith('#'): continue
+      field = line.strip().split()
+      if len(field) < 4: continue
+      if field[1] != component: continue
+      meta = {}
+      meta['master'] = field[0]
+      meta['sourcehost'] = field[2]
+      meta['sourcedir'] = field[3]
+      meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set()
+      meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set()
+      return meta
+    else:
+      return None
 
 cleanup_dirs = []
 def run_mirror(component):
+  meta = load_component_info(component)
+  if meta is None:
+    log("Component %s not found."%(component,))
+    return False
+  clients = allclients - meta['extraignoreclients']
+
   # setup
-  basemaster = os.path.join(base, 'master')
+  basemaster = os.path.join(config['base'], 'master')
   componentdir = os.path.join(basemaster, component)
   cur = componentdir + '-current-push'
   live = componentdir + '-current-live'
@@ -125,14 +174,25 @@ def run_mirror(component):
   os.chmod(tmpdir_new, 0755)
 
   locks = []
-  for p in (componentdir, live, tmpdir_new):
-    if not os.path.exists(p): os.mkdir(p, 0755)
+  lockfiles = [ os.path.join(basemaster, component + ".lock") ]
+  for p in lockfiles:
     fd = os.open(p, os.O_RDONLY)
     log("Acquiring lock for %s(%d)."%(p,fd))
     fcntl.flock(fd, fcntl.LOCK_EX)
     locks.append(fd)
   log("All locks acquired.")
 
+  for p in (live, ):
+    if not os.path.exists(p): os.mkdir(p, 0755)
+
+  #for p in (componentdir, live, tmpdir_new):
+  #  if not os.path.exists(p): os.mkdir(p, 0755)
+  #  fd = os.open(p, os.O_RDONLY)
+  #  log("Acquiring lock for %s(%d)."%(p,fd))
+  #  fcntl.flock(fd, fcntl.LOCK_EX)
+  #  locks.append(fd)
+  #log("All locks acquired.")
+
   serialfile = os.path.join(componentdir, serialname)
   try:
     with open(serialfile) as f: serial = int(f.read())
@@ -151,7 +211,7 @@ def run_mirror(component):
   log("Renaming %s to %s."%(tmpdir_new, cur))
   os.rename(tmpdir_new, cur)
 
-  proceed = callout(component, serial)
+  proceed = callout(component, serial, clients)
 
   if proceed:
     log("Moving %s aside."%(live,))
@@ -160,7 +220,8 @@ def run_mirror(component):
     os.rename(cur, live)
     log("Cleaning up.")
     shutil.rmtree(tmpdir_old)
-    log("Done.")
+    if had_warnings: log("Done, with warnings.")
+    else: log("Done.")
     ret = True
   else:
     log("Aborted.")