From 1552612184671215e066d620536e0594a2bb4ea5 Mon Sep 17 00:00:00 2001
From: Tollef Fog Heen <tfheen@err.no>
Date: Sat, 30 Sep 2017 21:00:19 +0200
Subject: [PATCH] Add health checking support for mirrors

Add a small daemon which checks if the local Last-Modified of a given
file is the same as on other hosts.  If it is, write a file saying
that we can receive traffic, else remove it.

Also map that file to /_health in the Apache config so bgpd/fastly can
check it.
---
 .../roles/files/mirror_health/mirror-health   | 52 +++++++++++++++++++
 modules/roles/manifests/debian_mirror.pp      |  6 +++
 modules/roles/manifests/mirror_health.pp      | 32 ++++++++++++
 .../roles/templates/apache-ftp.debian.org.erb |  1 +
 .../roles/templates/mirror-health.service.erb | 20 +++++++
 5 files changed, 111 insertions(+)
 create mode 100755 modules/roles/files/mirror_health/mirror-health
 create mode 100644 modules/roles/manifests/mirror_health.pp
 create mode 100644 modules/roles/templates/mirror-health.service.erb

diff --git a/modules/roles/files/mirror_health/mirror-health b/modules/roles/files/mirror_health/mirror-health
new file mode 100755
index 000000000..fb8241ba0
--- /dev/null
+++ b/modules/roles/files/mirror_health/mirror-health
@@ -0,0 +1,52 @@
+#! /usr/bin/python3
+
+import os
+import requests
+import time
+import calendar
+import logging
+from email.utils import parsedate
+logging.basicConfig(level=logging.INFO)
+
+HOSTS = os.environ['MIRROR_CHECK_HOSTS'].split()
+OUTPUT_DIR = "/run/dsa-mirror-health-{}".format(os.environ['MIRROR_CHECK_SERVICE'])
+HEALTH_FILE = os.path.join(OUTPUT_DIR, "health")
+URL = os.environ['MIRROR_CHECK_URL']
+INTERVAL = int(os.environ.get('MIRROR_CHECK_INTERVAL', '60'))
+
+latest_tz = 0
+
+def retrieve_from_host(host, url):
+    proxies = {
+        'http': 'http://{}:80'.format(host),
+        'https': 'http://{}:443'.format(host),
+    }
+    return requests.get(url, timeout=5, proxies=proxies)
+
+def last_modified(response):
+    lm = 0
+    if response.status_code == '200' and response.headers.get('last-modified'):
+        lm = calendar.timegm(parsedate(response.headers['last-modified']))
+    return lm
+
+while True:
+    start = time.time()
+    for host in HOSTS:
+        lm = last_modified(retrieve_from_host(host, URL))
+        logging.debug("lm for host %s: %s", host, lm)
+        if lm > latest_tz:
+            latest_tz = lm
+    local_lm = last_modified(retrieve_from_host('localhost', URL))
+    logging.debug("lm for localhost: %s", lm)
+    if latest_tz <= local_lm:
+        try:
+            logging.debug("considering myself unhealthy")
+            os.remove(HEALTH_FILE)
+        except OSError:
+            pass
+    else:
+        logging.debug("considering myself healthy")
+        open(HEALTH_FILE, 'w').write("OK")
+    sleep_time = start + INTERVAL - time.time()
+    logging.debug("sleeping for %d seconds", sleep_time)
+    time.sleep(sleep_time)
diff --git a/modules/roles/manifests/debian_mirror.pp b/modules/roles/manifests/debian_mirror.pp
index 1eae31085..cae863f58 100644
--- a/modules/roles/manifests/debian_mirror.pp
+++ b/modules/roles/manifests/debian_mirror.pp
@@ -39,4 +39,10 @@ class roles::debian_mirror {
 			target_address => $onion_v4_addr,
 		}
 	}
+
+        roles::mirror_health { 'ftp':
+		check_hosts   => getfromhash($site::roles, 'debian_mirror'),
+		check_service => 'ftp',
+		url           => 'http://debian.backend.mirrors.debian.org/debian/dists/sid/Release',
+        }
 }
diff --git a/modules/roles/manifests/mirror_health.pp b/modules/roles/manifests/mirror_health.pp
new file mode 100644
index 000000000..3b0d14982
--- /dev/null
+++ b/modules/roles/manifests/mirror_health.pp
@@ -0,0 +1,32 @@
+class roles::mirror_health (
+	$check_hosts    = [],
+	$check_service  = '',
+	$url            = '',
+	$check_interval = 60,
+) {
+	package { 'python3-requests':
+		ensure   => installed,
+	}
+
+        # XXX: avoid duplicating this?
+	file { '/usr/local/sbin/mirror-health':
+		source => 'puppet:///modules/roles/mirror_health/mirror-health',
+		owner  => 'root',
+		group  => 'root',
+		mode   => '0555',
+	}
+
+        file { "/etc/systemd/system/mirror-health-${check_service}.service":
+		owner   => 'root',
+		group   => 'root',
+		mode    => '0444',
+                content => template('roles/mirror-health.service.erb'),
+                notify  => Exec['systemctl daemon-reload'],
+        }
+
+        file { "/etc/systemd/system/multi-user.target.wants/mirror-health-${check_service}.service":
+		ensure => 'link',
+		target => "../mirror-health-${check_service}.service",
+		notify => Exec['systemctl daemon-reload'],
+	}
+}
diff --git a/modules/roles/templates/apache-ftp.debian.org.erb b/modules/roles/templates/apache-ftp.debian.org.erb
index 534c8b142..e9cd92365 100644
--- a/modules/roles/templates/apache-ftp.debian.org.erb
+++ b/modules/roles/templates/apache-ftp.debian.org.erb
@@ -17,6 +17,7 @@
 
 	RedirectMatch "^/$" /debian/
 	Alias /debian/ <%= @archive_root %>/
+	Alias /_health /run/dsa/mirror-health/ftp.debian.org
 
 	ErrorLog /var/log/apache2/ftp.debian.org-error.log
 	CustomLog /var/log/apache2/ftp.debian.org-access.log privacy
diff --git a/modules/roles/templates/mirror-health.service.erb b/modules/roles/templates/mirror-health.service.erb
new file mode 100644
index 000000000..575c958f3
--- /dev/null
+++ b/modules/roles/templates/mirror-health.service.erb
@@ -0,0 +1,20 @@
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git
+##
+
+[Unit]
+Description=Mirror health checking <%= @healthcheck_service %>
+
+[Service]
+ExecStart=/usr/local/sbin/mirror-health
+RuntimeDirectory=dsa-mirror-health-<%= @healthcheck_service %>
+DynamicUser=true
+
+Environment="MIRROR_CHECK_SERVICE=<%= @check_service %>"
+Environment="MIRROR_CHECK_URL=<%= @url %>"
+Environment="MIRROR_CHECK_HOSTS=<%= @check_hosts %>"
+Environment="MIRROR_CHECK_INTERVAL=<%= @check_interval %>"
+
+[Install]
+WantedBy=multi-user.target
-- 
2.20.1