Move debug to store/collect health checker

diff --git a/modules/mirror_health/files/mirror-health b/modules/mirror_health/files/mirror-health
new file mode 100755 (executable)
index 0000000..025382e
--- /dev/null
+++ b/modules/mirror_health/files/mirror-health
@@ -0,0 +1,80 @@
+#! /usr/bin/python3
+
+import os
+import requests
+import time
+import calendar
+import logging
+import subprocess
+from email.utils import parsedate
+logging.basicConfig(level=logging.INFO)
+
+if 'MIRROR_CHECK_HOSTS' in os.environ:
+    HOSTS = os.environ['MIRROR_CHECK_HOSTS'].split()
+else:
+    HOSTS = open(os.environ['MIRROR_CHECK_HOSTS_FILE']).read().split()
+
+OUTPUT_DIR = "/run/dsa-mirror-health-{}".format(os.environ['MIRROR_CHECK_SERVICE'])
+HEALTH_FILE = os.path.join(OUTPUT_DIR, "health")
+URL = os.environ['MIRROR_CHECK_URL']
+HEALTH_CHECK_URL = os.environ['MIRROR_CHECK_HEALTH_URL']
+INTERVAL = int(os.environ.get('MIRROR_CHECK_INTERVAL', '60'))
+
+def retrieve_from_host(host, url):
+    proxies = {
+        'http': 'http://{}:80'.format(host),
+        'https': 'http://{}:443'.format(host),
+    }
+    headers = {'User-Agent': 'mirror-health'}
+    return requests.get(url, headers=headers, timeout=5, proxies=proxies, allow_redirects=False)
+
+def last_modified(response):
+    lm = 0
+    if response.status_code == 200 and response.headers.get('last-modified'):
+        lm = calendar.timegm(parsedate(response.headers['last-modified']))
+    return lm
+
+def healthy(response):
+    if response.status_code == 200:
+        return True
+    return False
+
+def check_shutdown():
+    if subprocess.call(['dsa-is-shutdown-scheduled']) == 0:
+        logging.info("considering myself unhealthy, shutdown scheduled")
+        return False
+    return True
+
+def check_uptodate():
+    latest_ts = 0
+    for host in HOSTS:
+        try:
+            lm = last_modified(retrieve_from_host(host, URL))
+            logging.debug("lm for host %s: %s", host, lm)
+            if healthy(retrieve_from_host(host, HEALTH_CHECK_URL)):
+                latest_ts = max(latest_ts, lm)
+        except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
+            pass
+    try:
+        local_lm = last_modified(retrieve_from_host('localhost', URL))
+    except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
+        return False
+    logging.debug("lm for localhost: %s", local_lm)
+    if local_lm < latest_ts:
+        logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts)
+        return False
+    return True
+
+while True:
+    start = time.time()
+    if check_shutdown() and check_uptodate():
+        logging.info("considering myself healthy")
+        open(HEALTH_FILE, 'w').write("OK")
+    else:
+        try:
+            os.remove(HEALTH_FILE)
+        except OSError:
+            pass
+    sleep_time = start + INTERVAL - time.time()
+    logging.debug("sleeping for %d seconds", sleep_time)
+    time.sleep(sleep_time)

diff --git a/modules/mirror_health/manifests/init.pp b/modules/mirror_health/manifests/init.pp
new file mode 100644 (file)
index 0000000..f2e9cca
--- /dev/null
+++ b/modules/mirror_health/manifests/init.pp
@@ -0,0 +1,20 @@
+# base class for Debian's mirror-health checker
+class mirror_health () {
+  ensure_packages(['python3-requests'], { ensure => 'installed' })
+
+  $script = '/usr/local/sbin/mirror-health'
+  $confdir = '/etc/dsa/health-check'
+
+  file { $script:
+    source => 'puppet:///modules/mirror_health/mirror-health',
+    mode   => '0555',
+  }
+
+  file { $confdir:
+    ensure  => 'directory',
+    purge   => true,
+    force   => true,
+    recurse => true,
+    mode    => '0755';
+  }
+}

diff --git a/modules/mirror_health/manifests/service.pp b/modules/mirror_health/manifests/service.pp
new file mode 100644 (file)
index 0000000..312a63d
--- /dev/null
+++ b/modules/mirror_health/manifests/service.pp
@@ -0,0 +1,58 @@
+# base class for Debian's mirror-health checker
+define mirror_health::service (
+  String $this_host_service_name,
+  String $url,
+  String $health_url,
+  String $check_service = $name,
+  Integer $check_interval = 60,
+  Enum['present','absent'] $ensure = 'present',
+) {
+  include mirror_health
+
+  $service_file = "/etc/systemd/system/mirror-health-${check_service}.service"
+
+  file { $service_file:
+    content => template('mirror_health/mirror-health.service.erb'),
+    notify  => [Exec['systemctl daemon-reload'], Service["mirror-health-${check_service}"]],
+  }
+
+  $service_before = $ensure ? {
+    present => [],
+    default => [ File[$service_file], ],
+  }
+  $service_subscribe = $ensure ? {
+    present => [ File[$service_file], ],
+    default => [],
+  }
+
+  $ensure_service = $ensure ? {
+    present => running,
+    absent  => stopped,
+  }
+  $ensure_enable = $ensure ? {
+    present => true,
+    absent  => false,
+  }
+  service { "mirror-health-${check_service}":
+    ensure    => $ensure_service,
+    enable    => $ensure_enable,
+    require   => Exec['systemctl daemon-reload'],
+    before    => $service_before,
+    subscribe => $service_subscribe + [ File[ $mirror_health::script ] ],
+  }
+
+  $hosts_file = "${mirror_health::confdir}/${check_service}.hosts"
+  $tag = "mirror_health::service::${check_service}::hosts"
+  concat { $hosts_file:
+    ensure         => $ensure,
+    ensure_newline => true,
+    mode           => '0444',
+    notify         => Service["mirror-health-${check_service}"],
+  }
+  @@concat::fragment { "mirror_health::service::${check_service}::hosts::${this_host_service_name}":
+    tag     => $tag,
+    target  => $hosts_file,
+    content => $this_host_service_name,
+  }
+  Concat::Fragment <<| tag == $tag |>>
+}

diff --git a/modules/mirror_health/templates/mirror-health.service.erb b/modules/mirror_health/templates/mirror-health.service.erb
new file mode 100644 (file)
index 0000000..fd1c07b
--- /dev/null
+++ b/modules/mirror_health/templates/mirror-health.service.erb
@@ -0,0 +1,26 @@
+##
+## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE.
+## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git
+##
+
+[Unit]
+Description=Mirror health checking <%= @check_service %>
+
+[Service]
+ExecStart=/usr/local/sbin/mirror-health
+RuntimeDirectory=dsa-mirror-health-<%= @check_service %>
+ProtectSystem=full
+ProtectHome=read-only
+PrivateTmp=true
+User=nobody
+Group=nogroup
+Restart=always
+
+Environment="MIRROR_CHECK_SERVICE=<%= @check_service %>"
+Environment="MIRROR_CHECK_URL=<%= @url %>"
+Environment="MIRROR_CHECK_HEALTH_URL=<%= @health_url %>"
+Environment="MIRROR_CHECK_INTERVAL=<%= @check_interval %>"
+Environment="MIRROR_CHECK_HOSTS_FILE=/etc/dsa/health-check/<%= @check_service %>.hosts"
+
+[Install]
+WantedBy=multi-user.target

diff --git a/modules/roles/manifests/debug_mirror.pp b/modules/roles/manifests/debug_mirror.pp
index 0b361d8..38983c0 100644 (file)
--- a/modules/roles/manifests/debug_mirror.pp
+++ b/modules/roles/manifests/debug_mirror.pp
@@ -41,12 +41,9 @@ class roles::debug_mirror(
 
   Ferm::Rule::Simple <<| tag == 'ssh::server::from::ftp_master' |>>
 
-  $hosts_to_check = hiera('roles.debug_mirror', {})
-    .map |$h| { $h[1]['service-hostname'] }
-  roles::mirror_health { 'debug':
-    check_hosts   => $hosts_to_check,
-    check_service => 'debug',
-    url           => 'http://debug.backend.mirrors.debian.org/debian-debug/dists/sid-debug/Release',
-    health_url    => 'http://debug.backend.mirrors.debian.org/_health',
+  mirror_health::service { 'debug':
+    this_host_service_name => hiera('roles.debug_mirror')[$::fqdn]['service-hostname'],
+    url                    => 'http://debug.backend.mirrors.debian.org/debian-debug/dists/sid-debug/Release',
+    health_url             => 'http://debug.backend.mirrors.debian.org/_health',
   }
 }