create static-components.conf from a yaml file, re-add -anu, but let it mirror only www

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index e40a6f7..692e27e 100644 (file)
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -122,10 +122,12 @@ roles:
     - senfter.debian.org
     - mirror-csail.debian.org
     - mirror-isc.debian.org
+    - mirror-anu.debian.org
   # when adding a new static mirror, allow it to sync etc, but do not push to it and wait for it.  For this, also add it to static_mirror_nopush.
   static_mirror_nopush:
     - dummy
     - mirror-isc.debian.org
+    - mirror-anu.debian.org
   static_source:
     - master.debian.org
     - dillon.debian.org

diff --git a/modules/roles/files/static-mirroring/static-components.conf b/modules/roles/files/static-mirroring/static-components.conf
deleted file mode 100644 (file)
index 2ebb107..0000000
--- a/modules/roles/files/static-mirroring/static-components.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-# puppetd maintained
-# <master>             <service>                       <source host>                   <directory>                             <extra push hosts, comma separated>             <hosts to not mirror this component to>
-
-bizet.debian.org       bits.debian.org                 master.debian.org               /srv/bits-master.debian.org/htdocs
-bizet.debian.org       network-test.debian.org         master.debian.org               /srv/network-test.debian.org/htdocs
-bizet.debian.org       planet.debian.org               philp.debian.org                /srv/planet.debian.org/www
-bizet.debian.org       www.debian.org                  wolkenstein.debian.org          /srv/www.debian.org/www
-dillon.debian.org      appstream.debian.org            mekeel.debian.org               /srv/appstream.debian.org/public
-dillon.debian.org      blends.debian.org               dillon.debian.org               /srv/blends.debian.org/www
-dillon.debian.org      d-i.debian.org                  dillon.debian.org               /srv/d-i.debian.org/www
-dillon.debian.org      debaday.debian.net              dillon.debian.org               /srv/debaday.debian.net/htdocs
-dillon.debian.org      debdeltas.debian.net            donizetti.debian.org            /srv/debdelta.debian.org/www/debdeltas
-dillon.debian.org      dsa.debian.org                  dillon.debian.org               /srv/dsa.debian.org/htdocs
-dillon.debian.org      lintian.debian.org              lindsay.debian.org              /srv/lintian.debian.org/www
-dillon.debian.org      mozilla.debian.net              dillon.debian.org               /srv/mozilla.debian.net/htdocs
-dillon.debian.org      news.debian.net                 dillon.debian.org               /srv/news.debian.net/htdocs
-dillon.debian.org      rtc.debian.org                  dillon.debian.org               /srv/rtc.debian.org/htdocs
-dillon.debian.org      security-team.debian.org        dillon.debian.org               /srv/security-team.debian.org/htdocs
-dillon.debian.org      wnpp-by-tags.debian.net         dillon.debian.org               /srv/wnpp-by-tags.debian.net/www
-dillon.debian.org      www.ports.debian.org            dillon.debian.org               /srv/ports.debian.org/htdocs
-dillon.debian.org      timeline.debian.net             dillon.debian.org               /srv/timeline.debian.net/htdocs
-franck.debian.org      backports.debian.org            franck.debian.org               /srv/backports.debian.org/htdocs
-franck.debian.org      incoming.debian.org             franck.debian.org               /srv/ftp-master.debian.org/public/incoming.debian.org/web
-franck.debian.org      metadata.ftp-master.debian.org  franck.debian.org               /srv/ftp-master.debian.org/export

diff --git a/modules/roles/manifests/static_base.pp b/modules/roles/manifests/static_base.pp
index 216c9bf..8a92526 100644 (file)
--- a/modules/roles/manifests/static_base.pp
+++ b/modules/roles/manifests/static_base.pp
@@ -7,7 +7,7 @@ class roles::static_base {
        }
 
        file { '/etc/static-components.conf':
-               source => 'puppet:///modules/roles/static-mirroring/static-components.conf',
+               content => template('roles/static-mirroring/static-components.conf.erb'),
        }
 
        file { '/etc/ssh/userkeys/staticsync':

diff --git a/modules/roles/misc/static-components.yaml b/modules/roles/misc/static-components.yaml
new file mode 100644 (file)
index 0000000..31a71c9
--- /dev/null
+++ b/modules/roles/misc/static-components.yaml
@@ -0,0 +1,73 @@
+--- 
+
+components:
+  bits.debian.org:
+    master: bizet.debian.org
+    source: master.debian.org:/srv/bits-master.debian.org/htdocs
+  network-test.debian.org:
+    master: bizet.debian.org
+    source: master.debian.org:/srv/network-test.debian.org/htdocs
+  planet.debian.org:
+    master: bizet.debian.org
+    source: philp.debian.org:/srv/planet.debian.org/www
+  www.debian.org:
+    master: bizet.debian.org
+    source: wolkenstein.debian.org:/srv/www.debian.org/www
+  appstream.debian.org:
+    master: dillon.debian.org
+    source: mekeel.debian.org:/srv/appstream.debian.org/public
+  blends.debian.org:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/blends.debian.org/www
+  d-i.debian.org:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/d-i.debian.org/www
+  debaday.debian.net:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/debaday.debian.net/htdocs
+  debdeltas.debian.net:
+    master: dillon.debian.org
+    source: donizetti.debian.org:/srv/debdelta.debian.org/www/debdeltas
+  dsa.debian.org:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/dsa.debian.org/htdocs
+  lintian.debian.org:
+    master: dillon.debian.org
+    source: lindsay.debian.org:/srv/lintian.debian.org/www
+  mozilla.debian.net:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/mozilla.debian.net/htdocs
+  news.debian.net:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/news.debian.net/htdocs
+  rtc.debian.org:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/rtc.debian.org/htdocs
+  security-team.debian.org:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/security-team.debian.org/htdocs
+  wnpp-by-tags.debian.net:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/wnpp-by-tags.debian.net/www
+  www.ports.debian.org:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/ports.debian.org/htdocs
+  timeline.debian.net:
+    master: dillon.debian.org
+    source: dillon.debian.org:/srv/timeline.debian.net/htdocs
+  backports.debian.org:
+    master: franck.debian.org
+    source: franck.debian.org:/srv/backports.debian.org/htdocs
+  incoming.debian.org:
+    master: franck.debian.org
+    source: franck.debian.org:/srv/ftp-master.debian.org/public/incoming.debian.org/web
+  metadata.ftp-master.debian.org:
+    master: franck.debian.org
+    source: franck.debian.org:/srv/ftp-master.debian.org/export
+
+# The list of mirrors comes from puppet's hieradata.  This is just to set some arguments for specific mirrors.
+mirrors:
+  mirror-anu.debian.org:
+    components-include: # all others are thus excluded
+      - www.debian.org
+

diff --git a/modules/roles/templates/static-mirroring/static-components.conf.erb b/modules/roles/templates/static-mirroring/static-components.conf.erb
new file mode 100644 (file)
index 0000000..1656082
--- /dev/null
+++ b/modules/roles/templates/static-mirroring/static-components.conf.erb
@@ -0,0 +1,40 @@
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git
+##
+
+<%=
+
+lines = []
+
+lines << "# This file has been autogenerated and pushed by puppet.  Edit static-components.yaml in puppet."
+lines << "# <master> <service> <source host> <directory> <extra push hosts, comma separated> <hosts to not mirror this component to>"
+
+
+config = YAML.load(File.open('/etc/puppet/modules/roles/misc/static-components.yaml').read)
+mirrors = scope.lookupvar('site::roles')['static_mirror']
+
+config['components'].each_pair do |component, conf|
+       %w{exclude-mirrors extra-push}.each do |key|
+               conf[key] = [] unless conf.has_key?(key)
+       end
+
+       srchost, srcpath = conf['source'].split(':', 2)
+
+       config['mirrors'].each do |mirror, mc|
+               next unless mirrors.include?(mirror)
+
+               if mc.has_key?('components-include') and not mc['components-include'].include?(component)
+                       conf['exclude-mirrors'] << mirror
+               end
+       end
+
+       exclude = conf['exclude-mirrors'].join(',')
+       exclude = '-' unless exclude != ""
+       extrapush = conf['extra-push'].join(',')
+       extrapush = '-' unless extrapush != ""
+
+       lines << "#{conf['master']} #{component} #{srchost} #{srcpath} #{extrapush} #{exclude}"
+end
+lines.join("\n")
+%>