X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=modules%2Froles%2Ffiles%2Fstatic-mirroring%2Fstatic-master-run;h=95c9355a5ffc6172f550f8489a09c1ebbae47fd7;hb=5dc5997952b5754e99c246f2887057de4ad4b126;hp=d2f364d6ba791873e937c51d63bb92c31f524b91;hpb=63519a744307c2b8b063fb51002f48326a162d9a;p=mirror%2Fdsa-puppet.git

diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run
index d2f364d6b..95c9355a5 100755
--- a/modules/roles/files/static-mirroring/static-master-run
+++ b/modules/roles/files/static-mirroring/static-master-run
@@ -9,23 +9,36 @@ import sys
 import tempfile
 import time
 
-base="/srv/static.debian.org"
 serialname = '.serial'
 had_warnings = False
 
-clients = []
+conffile = '/etc/staticsync.conf'
+config={}
+
+with open(conffile) as f:
+  for line in f:
+    line = line.rstrip()
+    if not line or line.startswith("#"): continue
+    (name, value) = line.split("=")
+    config[name] = value
+
+for key in ('base',):
+  if not key in config:
+    raise Exception("Configuration element '%s' not found in config file %s"%(key, conffile))
+
+allclients = set()
 with open('/etc/static-clients.conf') as f:
   for line in f:
     line = line.strip()
     if line == "": continue
     if line.startswith('#'): continue
-    clients.append(line)
+    allclients.add(line)
 
 def log(m):
   t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
   print t, m
 
-def stage1(pipes, status):
+def stage1(pipes, status, clients):
   for c in clients:
     p = pipes[c]
     while 1:
@@ -71,7 +84,7 @@ def count_statuses(status):
     else: cnt[v] += 1
   return cnt
 
-def stage2(pipes, status, command):
+def stage2(pipes, status, command, clients):
   for c in clients:
     if status[c] != 'waiting': continue
     log("%s << %s"%(c, command))
@@ -86,7 +99,7 @@ def stage2(pipes, status, command):
       log("%s >> %s"%(c, l))
     log("%s: returned %d"%(c, p.returncode))
 
-def callout(component, serial):
+def callout(component, serial, clients):
   log("Calling clients...")
   pipes = {}
   status = {}
@@ -97,16 +110,16 @@ def callout(component, serial):
     status[c] = 'in-progress'
 
   log("Stage 1...")
-  stage1(pipes, status)
+  stage1(pipes, status, clients)
   log("Stage 1 done.")
   cnt = count_statuses(status)
 
   if 'failed' in cnt and cnt['failed'] >= 2:
     log("%d clients failed, aborting..."%(cnt['failed'],))
-    stage2(pipes, status, 'abort')
+    stage2(pipes, status, 'abort', clients)
     return False
 
-  failedmirrorsfile = os.path.join(base, 'master', component + "-failedmirrors")
+  failedmirrorsfile = os.path.join(config['base'], 'master', component + "-failedmirrors")
   if 'failed' in cnt:
     log("WARNING: %d clients failed!  Continuing anyway!"%(cnt['failed'],))
     global had_warnings
@@ -116,21 +129,43 @@ def callout(component, serial):
       if status[c] == 'failed': f.write(c+"\n")
     f.close()
   else:
-    os.unlink(failedmirrorsfile)
+    if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
 
   if 'waiting' in cnt:
     log("Committing...")
-    stage2(pipes, status, 'go')
+    stage2(pipes, status, 'go', clients)
     return True
   else:
     log("All clients up to date.")
     return True
 
+def load_component_info(component):
+  with open('/etc/static-components.conf') as f:
+    for line in f:
+      if line.startswith('#'): continue
+      field = line.strip().split()
+      if len(field) < 4: continue
+      if field[1] != component: continue
+      meta = {}
+      meta['master'] = field[0]
+      meta['sourcehost'] = field[2]
+      meta['sourcedir'] = field[3]
+      meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set()
+      meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set()
+      return meta
+    else:
+      return None
 
 cleanup_dirs = []
 def run_mirror(component):
+  meta = load_component_info(component)
+  if meta is None:
+    log("Component %s not found."%(component,))
+    return False
+  clients = allclients - meta['extraignoreclients']
+
   # setup
-  basemaster = os.path.join(base, 'master')
+  basemaster = os.path.join(config['base'], 'master')
   componentdir = os.path.join(basemaster, component)
   cur = componentdir + '-current-push'
   live = componentdir + '-current-live'
@@ -139,14 +174,25 @@ def run_mirror(component):
   os.chmod(tmpdir_new, 0755)
 
   locks = []
-  for p in (componentdir, live, tmpdir_new):
-    if not os.path.exists(p): os.mkdir(p, 0755)
+  lockfiles = [ os.path.join(basemaster, component + ".lock") ]
+  for p in lockfiles:
     fd = os.open(p, os.O_RDONLY)
     log("Acquiring lock for %s(%d)."%(p,fd))
     fcntl.flock(fd, fcntl.LOCK_EX)
     locks.append(fd)
   log("All locks acquired.")
 
+  for p in (live, ):
+    if not os.path.exists(p): os.mkdir(p, 0755)
+
+  #for p in (componentdir, live, tmpdir_new):
+  #  if not os.path.exists(p): os.mkdir(p, 0755)
+  #  fd = os.open(p, os.O_RDONLY)
+  #  log("Acquiring lock for %s(%d)."%(p,fd))
+  #  fcntl.flock(fd, fcntl.LOCK_EX)
+  #  locks.append(fd)
+  #log("All locks acquired.")
+
   serialfile = os.path.join(componentdir, serialname)
   try:
     with open(serialfile) as f: serial = int(f.read())
@@ -165,7 +211,7 @@ def run_mirror(component):
   log("Renaming %s to %s."%(tmpdir_new, cur))
   os.rename(tmpdir_new, cur)
 
-  proceed = callout(component, serial)
+  proceed = callout(component, serial, clients)
 
   if proceed:
     log("Moving %s aside."%(live,))