12 base="/srv/static.debian.org"
13 serialname = '.serial'
17 with open('/etc/static-clients.conf') as f:
20 if line == "": continue
21 if line.startswith('#'): continue
25 t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime())
28 def stage1(pipes, status, clients):
32 line = p.stdout.readline()
38 log("%s: failed with returncode %d"%(c,p.returncode))
42 log("%s >> %s"%(c, line))
43 if not line.startswith('[MSM]'): continue
44 kw = string.split(line, ' ', 2)[1]
46 if kw == 'ALREADY-CURRENT':
47 pipes[c].stdout.close()
48 pipes[c].stdin.close()
51 log("%s: already current"%(c,))
54 log("%s: said ALREADY-CURRENT but returncode %d"%(c,p.returncode))
57 elif kw == 'STAGE1-DONE':
58 log("%s: waiting"%(c,))
61 elif kw in ['STAGE1-START']:
64 log("%s: ignoring unknown line"%(c,))
66 def count_statuses(status):
70 if v not in cnt: cnt[v] = 1
74 def stage2(pipes, status, command, clients):
76 if status[c] != 'waiting': continue
77 log("%s << %s"%(c, command))
78 pipes[c].stdin.write("%s\n"%(command,))
81 if status[c] != 'waiting': continue
84 (o, dummy) = p.communicate('')
85 for l in string.split(o, "\n"):
86 log("%s >> %s"%(c, l))
87 log("%s: returned %d"%(c, p.returncode))
89 def callout(component, serial, clients):
90 log("Calling clients...")
94 args = ['ssh', '-o', 'BatchMode=yes', c, 'mirror', component, "%d"%(serial,)]
95 p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
97 status[c] = 'in-progress'
100 stage1(pipes, status, clients)
102 cnt = count_statuses(status)
104 if 'failed' in cnt and cnt['failed'] >= 2:
105 log("%d clients failed, aborting..."%(cnt['failed'],))
106 stage2(pipes, status, 'abort', clients)
109 failedmirrorsfile = os.path.join(base, 'master', component + "-failedmirrors")
111 log("WARNING: %d clients failed! Continuing anyway!"%(cnt['failed'],))
114 f = open(failedmirrorsfile, "w")
116 if status[c] == 'failed': f.write(c+"\n")
119 if os.path.exists(failedmirrorsfile): os.unlink(failedmirrorsfile)
123 stage2(pipes, status, 'go', clients)
126 log("All clients up to date.")
129 def load_component_info(component):
130 with open('/etc/static-components.conf') as f:
132 if line.startswith('#'): continue
133 field = line.strip().split()
134 if len(field) < 4: continue
135 if field[1] != component: continue
137 meta['master'] = field[0]
138 meta['sourcehost'] = field[2]
139 meta['sourcedir'] = field[3]
140 meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set()
141 meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set()
147 def run_mirror(component):
148 meta = load_component_info(component)
150 log("Component %s not found."%(component,))
152 clients = allclients - meta['extraignoreclients']
155 basemaster = os.path.join(base, 'master')
156 componentdir = os.path.join(basemaster, component)
157 cur = componentdir + '-current-push'
158 live = componentdir + '-current-live'
159 tmpdir_new = tempfile.mkdtemp(prefix=component+'-live.new-', dir=basemaster); cleanup_dirs.append(tmpdir_new);
160 tmpdir_old = tempfile.mkdtemp(prefix=component+'-live.old-', dir=basemaster); cleanup_dirs.append(tmpdir_old);
161 os.chmod(tmpdir_new, 0755)
164 for p in (componentdir, live, tmpdir_new):
165 if not os.path.exists(p): os.mkdir(p, 0755)
166 fd = os.open(p, os.O_RDONLY)
167 log("Acquiring lock for %s(%d)."%(p,fd))
168 fcntl.flock(fd, fcntl.LOCK_EX)
170 log("All locks acquired.")
172 serialfile = os.path.join(componentdir, serialname)
174 with open(serialfile) as f: serial = int(f.read())
176 serial = int(time.time())
177 with open(serialfile, "w") as f: f.write("%d\n"%(serial,))
178 log("Serial is %s."%(serial,))
180 log("Populating %s."%(tmpdir_new,))
181 subprocess.check_call(['cp', '-al', os.path.join(componentdir, '.'), tmpdir_new])
183 if os.path.exists(cur):
184 log("Removing existing %s."%(cur,))
187 log("Renaming %s to %s."%(tmpdir_new, cur))
188 os.rename(tmpdir_new, cur)
190 proceed = callout(component, serial, clients)
193 log("Moving %s aside."%(live,))
194 os.rename(live, os.path.join(tmpdir_old, 'old'))
195 log("Renaming %s to %s."%(cur, live))
198 shutil.rmtree(tmpdir_old)
199 if had_warnings: log("Done, with warnings.")
212 if len(sys.argv) != 2:
213 print >> sys.stderr, "Usage: %s <component>"%(sys.argv[0],)
215 component = sys.argv[1]
219 ok = run_mirror(component)
221 for p in cleanup_dirs:
222 if os.path.exists(p): shutil.rmtree(p)
228 # vim:set shiftwidth=2: