Add health checking support for mirrors

Add a small daemon which checks if the local Last-Modified of a given
file is the same as on other hosts.  If it is, write a file saying
that we can receive traffic, else remove it.

Also map that file to /_health in the Apache config so bgpd/fastly can
check it.

diff --git a/modules/roles/files/mirror_health/mirror-health b/modules/roles/files/mirror_health/mirror-health
new file mode 100755 (executable)
index 0000000..fb8241b
--- /dev/null
+++ b/modules/roles/files/mirror_health/mirror-health
@@ -0,0 +1,52 @@
+#! /usr/bin/python3
+
+import os
+import requests
+import time
+import calendar
+import logging
+from email.utils import parsedate
+logging.basicConfig(level=logging.INFO)
+
+HOSTS = os.environ['MIRROR_CHECK_HOSTS'].split()
+OUTPUT_DIR = "/run/dsa-mirror-health-{}".format(os.environ['MIRROR_CHECK_SERVICE'])
+HEALTH_FILE = os.path.join(OUTPUT_DIR, "health")
+URL = os.environ['MIRROR_CHECK_URL']
+INTERVAL = int(os.environ.get('MIRROR_CHECK_INTERVAL', '60'))
+
+latest_tz = 0
+
+def retrieve_from_host(host, url):
+    proxies = {
+        'http': 'http://{}:80'.format(host),
+        'https': 'http://{}:443'.format(host),
+    }
+    return requests.get(url, timeout=5, proxies=proxies)
+
+def last_modified(response):
+    lm = 0
+    if response.status_code == '200' and response.headers.get('last-modified'):
+        lm = calendar.timegm(parsedate(response.headers['last-modified']))
+    return lm
+
+while True:
+    start = time.time()
+    for host in HOSTS:
+        lm = last_modified(retrieve_from_host(host, URL))
+        logging.debug("lm for host %s: %s", host, lm)
+        if lm > latest_tz:
+            latest_tz = lm
+    local_lm = last_modified(retrieve_from_host('localhost', URL))
+    logging.debug("lm for localhost: %s", lm)
+    if latest_tz <= local_lm:
+        try:
+            logging.debug("considering myself unhealthy")
+            os.remove(HEALTH_FILE)
+        except OSError:
+            pass
+    else:
+        logging.debug("considering myself healthy")
+        open(HEALTH_FILE, 'w').write("OK")
+    sleep_time = start + INTERVAL - time.time()
+    logging.debug("sleeping for %d seconds", sleep_time)
+    time.sleep(sleep_time)

diff --git a/modules/roles/manifests/debian_mirror.pp b/modules/roles/manifests/debian_mirror.pp
index 1eae310..cae863f 100644 (file)
--- a/modules/roles/manifests/debian_mirror.pp
+++ b/modules/roles/manifests/debian_mirror.pp
@@ -39,4 +39,10 @@ class roles::debian_mirror {
                        target_address => $onion_v4_addr,
                }
        }
+
+        roles::mirror_health { 'ftp':
+               check_hosts   => getfromhash($site::roles, 'debian_mirror'),
+               check_service => 'ftp',
+               url           => 'http://debian.backend.mirrors.debian.org/debian/dists/sid/Release',
+        }
 }

diff --git a/modules/roles/manifests/mirror_health.pp b/modules/roles/manifests/mirror_health.pp
new file mode 100644 (file)
index 0000000..3b0d149
--- /dev/null
+++ b/modules/roles/manifests/mirror_health.pp
@@ -0,0 +1,32 @@
+class roles::mirror_health (
+       $check_hosts    = [],
+       $check_service  = '',
+       $url            = '',
+       $check_interval = 60,
+) {
+       package { 'python3-requests':
+               ensure   => installed,
+       }
+
+        # XXX: avoid duplicating this?
+       file { '/usr/local/sbin/mirror-health':
+               source => 'puppet:///modules/roles/mirror_health/mirror-health',
+               owner  => 'root',
+               group  => 'root',
+               mode   => '0555',
+       }
+
+        file { "/etc/systemd/system/mirror-health-${check_service}.service":
+               owner   => 'root',
+               group   => 'root',
+               mode    => '0444',
+                content => template('roles/mirror-health.service.erb'),
+                notify  => Exec['systemctl daemon-reload'],
+        }
+
+        file { "/etc/systemd/system/multi-user.target.wants/mirror-health-${check_service}.service":
+               ensure => 'link',
+               target => "../mirror-health-${check_service}.service",
+               notify => Exec['systemctl daemon-reload'],
+       }
+}

diff --git a/modules/roles/templates/apache-ftp.debian.org.erb b/modules/roles/templates/apache-ftp.debian.org.erb
index 534c8b1..e9cd923 100644 (file)
--- a/modules/roles/templates/apache-ftp.debian.org.erb
+++ b/modules/roles/templates/apache-ftp.debian.org.erb
@@ -17,6 +17,7 @@
 
        RedirectMatch "^/$" /debian/
        Alias /debian/ <%= @archive_root %>/
+       Alias /_health /run/dsa/mirror-health/ftp.debian.org
 
        ErrorLog /var/log/apache2/ftp.debian.org-error.log
        CustomLog /var/log/apache2/ftp.debian.org-access.log privacy

diff --git a/modules/roles/templates/mirror-health.service.erb b/modules/roles/templates/mirror-health.service.erb
new file mode 100644 (file)
index 0000000..575c958
--- /dev/null
+++ b/modules/roles/templates/mirror-health.service.erb
@@ -0,0 +1,20 @@
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git
+##
+
+[Unit]
+Description=Mirror health checking <%= @healthcheck_service %>
+
+[Service]
+ExecStart=/usr/local/sbin/mirror-health
+RuntimeDirectory=dsa-mirror-health-<%= @healthcheck_service %>
+DynamicUser=true
+
+Environment="MIRROR_CHECK_SERVICE=<%= @check_service %>"
+Environment="MIRROR_CHECK_URL=<%= @url %>"
+Environment="MIRROR_CHECK_HOSTS=<%= @check_hosts %>"
+Environment="MIRROR_CHECK_INTERVAL=<%= @check_interval %>"
+
+[Install]
+WantedBy=multi-user.target