From 0b6cb05ea797df93a05d5ba0d3ec549f16f7063d Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Sun, 22 Sep 2019 14:42:17 +0200 Subject: [PATCH] Move debug to store/collect health checker --- modules/mirror_health/files/mirror-health | 80 +++++++++++++++++++ modules/mirror_health/manifests/init.pp | 20 +++++ modules/mirror_health/manifests/service.pp | 58 ++++++++++++++ .../templates/mirror-health.service.erb | 26 ++++++ modules/roles/manifests/debug_mirror.pp | 11 +-- 5 files changed, 188 insertions(+), 7 deletions(-) create mode 100755 modules/mirror_health/files/mirror-health create mode 100644 modules/mirror_health/manifests/init.pp create mode 100644 modules/mirror_health/manifests/service.pp create mode 100644 modules/mirror_health/templates/mirror-health.service.erb diff --git a/modules/mirror_health/files/mirror-health b/modules/mirror_health/files/mirror-health new file mode 100755 index 000000000..025382e78 --- /dev/null +++ b/modules/mirror_health/files/mirror-health @@ -0,0 +1,80 @@ +#! /usr/bin/python3 + +import os +import requests +import time +import calendar +import logging +import subprocess +from email.utils import parsedate +logging.basicConfig(level=logging.INFO) + +if 'MIRROR_CHECK_HOSTS' in os.environ: + HOSTS = os.environ['MIRROR_CHECK_HOSTS'].split() +else: + HOSTS = open(os.environ['MIRROR_CHECK_HOSTS_FILE']).read().split() + +OUTPUT_DIR = "/run/dsa-mirror-health-{}".format(os.environ['MIRROR_CHECK_SERVICE']) +HEALTH_FILE = os.path.join(OUTPUT_DIR, "health") +URL = os.environ['MIRROR_CHECK_URL'] +HEALTH_CHECK_URL = os.environ['MIRROR_CHECK_HEALTH_URL'] +INTERVAL = int(os.environ.get('MIRROR_CHECK_INTERVAL', '60')) + +def retrieve_from_host(host, url): + proxies = { + 'http': 'http://{}:80'.format(host), + 'https': 'http://{}:443'.format(host), + } + headers = {'User-Agent': 'mirror-health'} + return requests.get(url, headers=headers, timeout=5, proxies=proxies, allow_redirects=False) + +def last_modified(response): + lm = 0 + if response.status_code == 200 and response.headers.get('last-modified'): + lm = calendar.timegm(parsedate(response.headers['last-modified'])) + return lm + +def healthy(response): + if response.status_code == 200: + return True + return False + +def check_shutdown(): + if subprocess.call(['dsa-is-shutdown-scheduled']) == 0: + logging.info("considering myself unhealthy, shutdown scheduled") + return False + return True + +def check_uptodate(): + latest_ts = 0 + for host in HOSTS: + try: + lm = last_modified(retrieve_from_host(host, URL)) + logging.debug("lm for host %s: %s", host, lm) + if healthy(retrieve_from_host(host, HEALTH_CHECK_URL)): + latest_ts = max(latest_ts, lm) + except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError): + pass + try: + local_lm = last_modified(retrieve_from_host('localhost', URL)) + except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError): + return False + logging.debug("lm for localhost: %s", local_lm) + if local_lm < latest_ts: + logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts) + return False + return True + +while True: + start = time.time() + if check_shutdown() and check_uptodate(): + logging.info("considering myself healthy") + open(HEALTH_FILE, 'w').write("OK") + else: + try: + os.remove(HEALTH_FILE) + except OSError: + pass + sleep_time = start + INTERVAL - time.time() + logging.debug("sleeping for %d seconds", sleep_time) + time.sleep(sleep_time) diff --git a/modules/mirror_health/manifests/init.pp b/modules/mirror_health/manifests/init.pp new file mode 100644 index 000000000..f2e9cca19 --- /dev/null +++ b/modules/mirror_health/manifests/init.pp @@ -0,0 +1,20 @@ +# base class for Debian's mirror-health checker +class mirror_health () { + ensure_packages(['python3-requests'], { ensure => 'installed' }) + + $script = '/usr/local/sbin/mirror-health' + $confdir = '/etc/dsa/health-check' + + file { $script: + source => 'puppet:///modules/mirror_health/mirror-health', + mode => '0555', + } + + file { $confdir: + ensure => 'directory', + purge => true, + force => true, + recurse => true, + mode => '0755'; + } +} diff --git a/modules/mirror_health/manifests/service.pp b/modules/mirror_health/manifests/service.pp new file mode 100644 index 000000000..312a63db5 --- /dev/null +++ b/modules/mirror_health/manifests/service.pp @@ -0,0 +1,58 @@ +# base class for Debian's mirror-health checker +define mirror_health::service ( + String $this_host_service_name, + String $url, + String $health_url, + String $check_service = $name, + Integer $check_interval = 60, + Enum['present','absent'] $ensure = 'present', +) { + include mirror_health + + $service_file = "/etc/systemd/system/mirror-health-${check_service}.service" + + file { $service_file: + content => template('mirror_health/mirror-health.service.erb'), + notify => [Exec['systemctl daemon-reload'], Service["mirror-health-${check_service}"]], + } + + $service_before = $ensure ? { + present => [], + default => [ File[$service_file], ], + } + $service_subscribe = $ensure ? { + present => [ File[$service_file], ], + default => [], + } + + $ensure_service = $ensure ? { + present => running, + absent => stopped, + } + $ensure_enable = $ensure ? { + present => true, + absent => false, + } + service { "mirror-health-${check_service}": + ensure => $ensure_service, + enable => $ensure_enable, + require => Exec['systemctl daemon-reload'], + before => $service_before, + subscribe => $service_subscribe + [ File[ $mirror_health::script ] ], + } + + $hosts_file = "${mirror_health::confdir}/${check_service}.hosts" + $tag = "mirror_health::service::${check_service}::hosts" + concat { $hosts_file: + ensure => $ensure, + ensure_newline => true, + mode => '0444', + notify => Service["mirror-health-${check_service}"], + } + @@concat::fragment { "mirror_health::service::${check_service}::hosts::${this_host_service_name}": + tag => $tag, + target => $hosts_file, + content => $this_host_service_name, + } + Concat::Fragment <<| tag == $tag |>> +} diff --git a/modules/mirror_health/templates/mirror-health.service.erb b/modules/mirror_health/templates/mirror-health.service.erb new file mode 100644 index 000000000..fd1c07b1d --- /dev/null +++ b/modules/mirror_health/templates/mirror-health.service.erb @@ -0,0 +1,26 @@ +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## + +[Unit] +Description=Mirror health checking <%= @check_service %> + +[Service] +ExecStart=/usr/local/sbin/mirror-health +RuntimeDirectory=dsa-mirror-health-<%= @check_service %> +ProtectSystem=full +ProtectHome=read-only +PrivateTmp=true +User=nobody +Group=nogroup +Restart=always + +Environment="MIRROR_CHECK_SERVICE=<%= @check_service %>" +Environment="MIRROR_CHECK_URL=<%= @url %>" +Environment="MIRROR_CHECK_HEALTH_URL=<%= @health_url %>" +Environment="MIRROR_CHECK_INTERVAL=<%= @check_interval %>" +Environment="MIRROR_CHECK_HOSTS_FILE=/etc/dsa/health-check/<%= @check_service %>.hosts" + +[Install] +WantedBy=multi-user.target diff --git a/modules/roles/manifests/debug_mirror.pp b/modules/roles/manifests/debug_mirror.pp index 0b361d8c8..38983c0c0 100644 --- a/modules/roles/manifests/debug_mirror.pp +++ b/modules/roles/manifests/debug_mirror.pp @@ -41,12 +41,9 @@ class roles::debug_mirror( Ferm::Rule::Simple <<| tag == 'ssh::server::from::ftp_master' |>> - $hosts_to_check = hiera('roles.debug_mirror', {}) - .map |$h| { $h[1]['service-hostname'] } - roles::mirror_health { 'debug': - check_hosts => $hosts_to_check, - check_service => 'debug', - url => 'http://debug.backend.mirrors.debian.org/debian-debug/dists/sid-debug/Release', - health_url => 'http://debug.backend.mirrors.debian.org/_health', + mirror_health::service { 'debug': + this_host_service_name => hiera('roles.debug_mirror')[$::fqdn]['service-hostname'], + url => 'http://debug.backend.mirrors.debian.org/debian-debug/dists/sid-debug/Release', + health_url => 'http://debug.backend.mirrors.debian.org/_health', } } -- 2.20.1