From a6a56555039941b10af2d856138661f50680485b Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Wed, 14 Jan 2015 22:43:28 +0100 Subject: [PATCH] Try porting torproject changes: support excluding mirror for a single static component --- .../static-mirroring/static-components.conf | 2 +- .../files/static-mirroring/static-master-run | 40 ++++++++++++++---- .../static-mirroring/static-mirror-run-all | 41 +++++++++++++++++++ modules/roles/manifests/static_mirror.pp | 7 +++- 4 files changed, 79 insertions(+), 11 deletions(-) create mode 100755 modules/roles/files/static-mirroring/static-mirror-run-all diff --git a/modules/roles/files/static-mirroring/static-components.conf b/modules/roles/files/static-mirroring/static-components.conf index 978503642..12b67db43 100644 --- a/modules/roles/files/static-mirroring/static-components.conf +++ b/modules/roles/files/static-mirroring/static-components.conf @@ -1,5 +1,5 @@ # puppetd maintained -# +# bizet.debian.org bits.debian.org master.debian.org /srv/bits-master.debian.org/htdocs bizet.debian.org lintian.debian.org lilburn.debian.org /srv/lintian.debian.org/www diff --git a/modules/roles/files/static-mirroring/static-master-run b/modules/roles/files/static-mirroring/static-master-run index c85e98447..e2a583e9e 100755 --- a/modules/roles/files/static-mirroring/static-master-run +++ b/modules/roles/files/static-mirroring/static-master-run @@ -13,19 +13,19 @@ base="/srv/static.debian.org" serialname = '.serial' had_warnings = False -clients = [] +allclients = set() with open('/etc/static-clients.conf') as f: for line in f: line = line.strip() if line == "": continue if line.startswith('#'): continue - clients.append(line) + allclients.add(line) def log(m): t = time.strftime("[%Y-%m-%d %H:%M:%S]", time.gmtime()) print t, m -def stage1(pipes, status): +def stage1(pipes, status, clients): for c in clients: p = pipes[c] while 1: @@ -71,7 +71,7 @@ def count_statuses(status): else: cnt[v] += 1 return cnt -def stage2(pipes, status, command): +def stage2(pipes, status, command, clients): for c in clients: if status[c] != 'waiting': continue log("%s << %s"%(c, command)) @@ -86,7 +86,7 @@ def stage2(pipes, status, command): log("%s >> %s"%(c, l)) log("%s: returned %d"%(c, p.returncode)) -def callout(component, serial): +def callout(component, serial, clients): log("Calling clients...") pipes = {} status = {} @@ -97,13 +97,13 @@ def callout(component, serial): status[c] = 'in-progress' log("Stage 1...") - stage1(pipes, status) + stage1(pipes, status, clients) log("Stage 1 done.") cnt = count_statuses(status) if 'failed' in cnt and cnt['failed'] >= 2: log("%d clients failed, aborting..."%(cnt['failed'],)) - stage2(pipes, status, 'abort') + stage2(pipes, status, 'abort', clients) return False failedmirrorsfile = os.path.join(base, 'master', component + "-failedmirrors") @@ -120,15 +120,37 @@ def callout(component, serial): if 'waiting' in cnt: log("Committing...") - stage2(pipes, status, 'go') + stage2(pipes, status, 'go', clients) return True else: log("All clients up to date.") return True +def load_component_info(component): + with open('/etc/static-components.conf') as f: + for line in f: + if line.startswith('#'): continue + field = line.strip().split() + if len(field) < 4: continue + if field[1] != component: continue + meta = {} + meta['master'] = field[0] + meta['sourcehost'] = field[2] + meta['sourcedir'] = field[3] + meta['extrapushhosts'] = set(field[4].split(',')) if len(field) > 4 else set() + meta['extraignoreclients'] = set(field[5].split(',')) if len(field) > 5 else set() + return meta + else: + return None cleanup_dirs = [] def run_mirror(component): + meta = load_component_info(component) + if meta is None: + log("Component %s not found."%(component,)) + return False + clients = allclients - meta['extraignoreclients'] + # setup basemaster = os.path.join(base, 'master') componentdir = os.path.join(basemaster, component) @@ -165,7 +187,7 @@ def run_mirror(component): log("Renaming %s to %s."%(tmpdir_new, cur)) os.rename(tmpdir_new, cur) - proceed = callout(component, serial) + proceed = callout(component, serial, clients) if proceed: log("Moving %s aside."%(live,)) diff --git a/modules/roles/files/static-mirroring/static-mirror-run-all b/modules/roles/files/static-mirroring/static-mirror-run-all new file mode 100755 index 000000000..11ec2884f --- /dev/null +++ b/modules/roles/files/static-mirroring/static-mirror-run-all @@ -0,0 +1,41 @@ +#!/bin/bash + +# initiate a mirror run for all components on this host. + +# Copyright (c) 2012, 2015 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +set -e +set -u + +awk -v host="$(hostname -f)" ' + !/^ *(#|$)/ { + split($6,ignorehosts,",") + for (i in ignorehosts) { + if (host == ignorehosts[i]) { + next + } + } + print $1, $2 + }' /etc/static-components.conf | + while read master component ; do + static-mirror-run --one-stage "/srv/static.debian.org/mirrors/$component" "$master:$component/-live-" + done diff --git a/modules/roles/manifests/static_mirror.pp b/modules/roles/manifests/static_mirror.pp index 3a614a93e..dd76b1cda 100644 --- a/modules/roles/manifests/static_mirror.pp +++ b/modules/roles/manifests/static_mirror.pp @@ -15,6 +15,11 @@ class roles::static_mirror { mode => '0555', } + file { '/usr/local/bin/static-mirror-run-all': + source => 'puppet:///modules/roles/static-mirroring/static-mirror-run-all', + mode => '0555', + } + file { '/srv/static.debian.org': ensure => directory, owner => staticsync, @@ -23,7 +28,7 @@ class roles::static_mirror { } file { '/etc/cron.d/puppet-static-mirror': - content => "PATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; awk '!/^ *(#|$)/ {print \$1, \$2}' /etc/static-components.conf | while read master component; do static-mirror-run --one-stage /srv/static.debian.org/mirrors/\$component \"\$master:\$component/-live-\" > /dev/null; done\n", + content => "MAILTO=root\nPATH=/usr/local/bin:/usr/bin:/bin\n@reboot staticsync sleep 60; chronic static-mirror-run-all\n", } $vhost_listen = $::hostname ? { -- 2.20.1