From: Peter Palfrader <peter@palfrader.org>
Date: Sat, 14 Sep 2019 14:22:27 +0000 (+0200)
Subject: Make staticsync a module and update references
X-Git-Url: https://git.adam-barratt.org.uk/?a=commitdiff_plain;h=a02c0d91f3ddf38694d750e8e8320bc2987e8365;p=mirror%2Fdsa-puppet.git

Make staticsync a module and update references
---

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 5bcf861fd..e18cf6899 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -27,6 +27,9 @@ paths:
   auto_clientcerts_dir: '/srv/puppet.debian.org/ca/RESULT/clientcerts'
 apt::sources::debian::location: 'https://deb.debian.org/debian/'
 
+staticsync::user: 'staticsync'
+staticsync::basedir: '/srv/static.debian.org'
+
 # all of these should be retired in favour of including the class role
 # with the host. weasel, 2019-09
 roles:
diff --git a/modules/roles/manifests/static_master.pp b/modules/roles/manifests/static_master.pp
new file mode 100644
index 000000000..2f786a4aa
--- /dev/null
+++ b/modules/roles/manifests/static_master.pp
@@ -0,0 +1,5 @@
+#
+class roles::static_master(
+) {
+  include staticsync::static_master
+}
diff --git a/modules/roles/manifests/static_mirror.pp b/modules/roles/manifests/static_mirror.pp
new file mode 100644
index 000000000..cf252d4ba
--- /dev/null
+++ b/modules/roles/manifests/static_mirror.pp
@@ -0,0 +1,5 @@
+# a static mirror.  It will get content for all components that it servers
+# from each component's master.
+class roles::static_mirror () {
+  include staticsync::static_mirror
+}
diff --git a/modules/roles/manifests/static_source.pp b/modules/roles/manifests/static_source.pp
new file mode 100644
index 000000000..3e66d3432
--- /dev/null
+++ b/modules/roles/manifests/static_source.pp
@@ -0,0 +1,4 @@
+class roles::static_source(
+) {
+  include staticsync::static_source
+}
diff --git a/modules/staticsync/manifests/base.pp b/modules/staticsync/manifests/base.pp
index 89a0a383c..3f6cd927c 100644
--- a/modules/staticsync/manifests/base.pp
+++ b/modules/staticsync/manifests/base.pp
@@ -1,31 +1,37 @@
 # the base class defining things common for all three static classes (master, mirror, source)
-class roles::static::base {
-  $query = 'nodes[certname] { resources { type = "Class" and title = "Roles::Static_mirror" } }'
+class staticsync::base inherits staticsync {
+
+  # fetch the list of static mirrors from PuppetDB
+  #
+  # This list is necessary so we can both do "include only these
+  # mirrors" but also "exclude these mirrors and include all others"
+  # in the config.
+  $query = 'nodes[certname] { resources { type = "Class" and title = "Staticsync::Static_mirror" } }'
   $static_mirrors = sort(puppetdb_query($query).map |$value| { $value["certname"] })
 
   file { '/etc/static-components.conf':
-    content => template('roles/static-mirroring/static-components.conf.erb'),
+    content => template('staticsync/static-components.conf.erb'),
   }
 
   file { '/usr/local/bin/staticsync-ssh-wrap':
-    source => 'puppet:///modules/roles/static-mirroring/staticsync-ssh-wrap',
+    source => 'puppet:///modules/staticsync/staticsync-ssh-wrap',
     mode   => '0555',
   }
 
   file { '/usr/local/bin/static-update-component':
-    source => 'puppet:///modules/roles/static-mirroring/static-update-component',
-    mode    => '0555',
+    source => 'puppet:///modules/staticsync/static-update-component',
+    mode   => '0555',
   }
 
   file { '/etc/staticsync.conf':
     content  => @("EOF"),
-                # This file is sourced by bash
-                # and parsed by python
-                #  - empty lines and lines starting with a # are ignored.
-                #  - other lines are key=value.  No extra spaces anywhere.  No quoting.
-                base=/srv/static.debian.org
-                masterbase=/home/staticsync/static-master/master
-                staticuser=staticsync
-                | EOF
+        # This file is sourced by bash
+        # and parsed by python
+        #  - empty lines and lines starting with a # are ignored.
+        #  - other lines are key=value.  No extra spaces anywhere.  No quoting.
+        base=${staticsync::basedir}
+        masterbase=${staticsync::basedir}/master
+        staticuser=${staticsync::user}
+        | EOF
   }
 }
diff --git a/modules/staticsync/manifests/init.pp b/modules/staticsync/manifests/init.pp
new file mode 100644
index 000000000..7b8a106c6
--- /dev/null
+++ b/modules/staticsync/manifests/init.pp
@@ -0,0 +1,15 @@
+# A base class for staticsync.  This owns the configuration variables, which
+# should be set by hiera.
+class staticsync (
+  String $user,
+  String $basedir,
+  # for ssh/firewalling purposes
+  Array[Stdlib::IP::Address] $public_addresses = $base::public_addresses,
+) {
+  if ! $user {
+    fail('Variable $user is not set.  Please provide a value (for instance via hiera).')
+  }
+  if ! $basedir {
+    fail('Variable $basedir is not set.  Please provider a value (for instance via hiera).')
+  }
+}
diff --git a/modules/staticsync/manifests/srvdir.pp b/modules/staticsync/manifests/srvdir.pp
index 7fef3830c..23eb6141e 100644
--- a/modules/staticsync/manifests/srvdir.pp
+++ b/modules/staticsync/manifests/srvdir.pp
@@ -1,13 +1,14 @@
 # create the directory on static hosts and disable backups
-class roles::static::srvdir {
-  file { '/srv/static.debian.org':
+class staticsync::srvdir (
+) {
+  file { $staticsync::basedir:
     ensure => directory,
     mode   => '0755',
-    owner  => 'staticsync',
-    group  => 'staticsync',
+    owner  => $staticsync::user,
+    group  => $staticsync::user,
   }
 
-  file { '/srv/static.debian.org/.nobackup':
+  file { "${staticsync::basedir}/.nobackup":
     content => '',
   }
 }
diff --git a/modules/staticsync/manifests/ssh.pp b/modules/staticsync/manifests/ssh.pp
index 22078cb60..d3373f03f 100644
--- a/modules/staticsync/manifests/ssh.pp
+++ b/modules/staticsync/manifests/ssh.pp
@@ -1,20 +1,20 @@
 # wrapper for ssh setup for statichosts
-class roles::static::ssh(
+class staticsync::ssh  (
   Variant[Array[String], String] $add_tag,
   String                         $collect_tag,
-  )
-{
-  ssh::keygen {'staticsync': }
+) {
+  ssh::keygen { $staticsync::user : }
 
-  ssh::authorized_key_add { 'staticsync':
-    target_user => 'staticsync',
+  ssh::authorized_key_add { "staticsync-${staticsync::user}":
+    target_user => $staticsync::user,
+    key         => dig($facts, 'ssh_keys_users', $staticsync::user, 'id_rsa.pub', 'line'),
     command     => "/usr/local/bin/staticsync-ssh-wrap ${::fqdn}",
-    key         => $facts['staticsync_key'],
-    options     => ['restrict', 'pty'],
+    options     => ['restrict','pty'],
+    from        => $staticsync::public_addresses,
     collect_tag => $add_tag,
   }
-  ssh::authorized_key_collect { 'staticsync':
-    target_user => 'staticsync',
+  ssh::authorized_key_collect { "staticsync-${staticsync::user}":
+    target_user => $staticsync::user,
     collect_tag => $collect_tag,
   }
 }
diff --git a/modules/staticsync/manifests/static_master.pp b/modules/staticsync/manifests/static_master.pp
index 6a36ed3fc..17efe762f 100644
--- a/modules/staticsync/manifests/static_master.pp
+++ b/modules/staticsync/manifests/static_master.pp
@@ -3,36 +3,44 @@
 # each component defines exactly one static master.  Content is copied from the source host
 # to the master, and from there to all the mirrors.
 #
-class roles::static_master {
-  include roles::static::base
-  include roles::static::srvdir
+class staticsync::static_master (
+) {
+  include staticsync::base
+  include staticsync::srvdir
 
   # masters need to talk to mirrors and sources and themselves
-  class { 'roles::static::ssh':
+  class { 'staticsync::ssh':
     add_tag     => [ 'staticsync-mirror', 'staticsync-source', 'staticsync-master' ],
     collect_tag => 'staticsync-master',
   }
 
   file { '/usr/local/bin/static-master-run':
-    source => 'puppet:///modules/roles/static-mirroring/static-master-run',
+    source => 'puppet:///modules/staticsync/static-master-run',
     mode   => '0555',
   }
   file { '/usr/local/bin/static-master-update-component':
-    source => 'puppet:///modules/roles/static-mirroring/static-master-update-component',
+    source => 'puppet:///modules/staticsync/static-master-update-component',
     mode   => '0555',
   }
   file { '/etc/static-clients.conf':
-    content => template('roles/static-mirroring/static-clients.conf.erb'),
+    content => template('staticsync/static-clients.conf.erb'),
   }
+#  concat { '/etc/static-clients.conf':
+#    ensure_newline => true,
+#    warn           => @(EOF),
+#    ## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+#    | EOF
+#  }
+#  Concat::Fragment <<| tag == 'staticsync::static-mirrors-to-trigger' |>>
 
-  file { '/home/staticsync/static-master':
+  file { "/home/${staticsync::user}/static-master":
     ensure => link,
-    target => '/srv/static.debian.org',
+    target => $staticsync::basedir,
   }
-  file { '/srv/static.debian.org/master':
+  file { "${staticsync::basedir}/master":
     ensure => directory,
     mode   => '0755',
-    owner  => 'staticsync',
-    group  => 'staticsync',
+    owner  => $staticsync::user,
+    group  => $staticsync::user,
   }
 }
diff --git a/modules/staticsync/manifests/static_mirror.pp b/modules/staticsync/manifests/static_mirror.pp
index 380ad3dac..bcec3110e 100644
--- a/modules/staticsync/manifests/static_mirror.pp
+++ b/modules/staticsync/manifests/static_mirror.pp
@@ -1,31 +1,47 @@
-# a static mirror
+# a static mirror.  It will get content for all components that it servers
+# from each component's master.
 #
-# this receives pushes from the master and then usually serves the content to the public
-class roles::static_mirror {
-  include roles::static::base
-  include roles::static::srvdir
+# Static mirrors register themselves via a stored-conf /etc/static-clients.conf
+# that the masters collect.  Mirrors can always pull, but they only get
+# triggered if they are in /etc/static-clients.conf.  So during initial
+# setup it might make sense to not get triggered and thus block updates of
+# components until all mirrors are in sync; set $get_triggered = false for that.
+#
+class staticsync::static_mirror (
+  Boolean $get_triggered = true,
+) {
+  include staticsync::base
+  include staticsync::srvdir
 
   # mirrors talk only to masters
-  class { 'roles::static::ssh':
+  class { 'staticsync::ssh':
     add_tag     => 'staticsync-master',
     collect_tag => 'staticsync-mirror',
   }
 
   file { '/usr/local/bin/static-mirror-run':
-    source => 'puppet:///modules/roles/static-mirroring/static-mirror-run',
+    source => 'puppet:///modules/staticsync/static-mirror-run',
     mode   => '0555',
   }
-
   file { '/usr/local/bin/static-mirror-run-all':
-    source => 'puppet:///modules/roles/static-mirroring/static-mirror-run-all',
+    source => 'puppet:///modules/staticsync/static-mirror-run-all',
     mode   => '0555',
   }
-
   file { '/etc/cron.d/puppet-static-mirror': ensure => absent, }
   concat::fragment { 'puppet-crontab--static-mirror':
     target => '/etc/cron.d/puppet-crontab',
-    content  => @(EOF)
-      @reboot staticsync sleep 60; chronic static-mirror-run-all
-      | EOF
+    content  => @("EOF"),
+        MAILTO=root
+        PATH=/usr/local/bin:/usr/bin:/bin
+        @reboot ${staticsync::user} sleep 60; chronic static-mirror-run-all
+        | EOF
+  }
+
+  if $get_triggered {
+    @@concat::fragment { "static-client-${::fqdn}":
+      tag     => 'staticsync::static-mirrors-to-trigger',
+      content => $::fqdn,
+      target  => '/etc/static-clients.conf',
+    }
   }
 }
diff --git a/modules/staticsync/manifests/static_source.pp b/modules/staticsync/manifests/static_source.pp
index 960366cf9..02e993845 100644
--- a/modules/staticsync/manifests/static_source.pp
+++ b/modules/staticsync/manifests/static_source.pp
@@ -1,17 +1,20 @@
-# a static source
+# Sources are systems that produce content that is then shipped to masters (which ship it to mirrors).
+# Source can trigger these mirror pulses.
 #
-# origin of static content.  From here it goes to the static master before that one pushes it to the mirrors
-class roles::static_source {
-  include roles::static::base
+# the class doesn't need anything beyond what is in the common base set for now.
+class staticsync::static_source (
+) {
+  include staticsync::base
 
-  if ! defined(Class["roles::static_master"]) {
+  if ! defined(Class["staticsync::static_master"]) {
     # sources talk only to masters, but only set this up if we are not
-    # *also* a static master since we cannot have two meaningful roles::static:ssh
+    # *also* a static master since we cannot have two meaningful staticsync:ssh
     # instances in the current setup.
     #
-    # this adds the limitation that the master of any component whose source is also a
-    # master node needs to have that same host as its master and not some other one.
-    class { 'roles::static::ssh':
+    # this adds the limitation that if a system is both master and source, then
+    # any components which have this system as a source need to also use the same
+    # system as their master.
+    class { 'staticsync::ssh':
       add_tag     => 'staticsync-master',
       collect_tag => 'staticsync-source',
     }
diff --git a/modules/staticsync/templates/static-components.conf.erb b/modules/staticsync/templates/static-components.conf.erb
index c04a3a56d..ec058411a 100644
--- a/modules/staticsync/templates/static-components.conf.erb
+++ b/modules/staticsync/templates/static-components.conf.erb
@@ -1,6 +1,5 @@
 ##
 ## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
-## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git
 ##
 
 <%=