modules/mirror_health/files/mirror-health

   1 #! /usr/bin/python3
   2
   3 import os
   4 import requests
   5 import time
   6 import calendar
   7 import logging
   8 import subprocess
   9 from email.utils import parsedate
  10 import urllib3.util.connection
  11
  12 logging.basicConfig(level=logging.INFO)
  13
  14 if 'MIRROR_CHECK_HOSTS' in os.environ:
  15     HOSTS = os.environ['MIRROR_CHECK_HOSTS'].split()
  16 else:
  17     HOSTS = open(os.environ['MIRROR_CHECK_HOSTS_FILE']).read().split()
  18
  19 OUTPUT_DIR = "/run/dsa-mirror-health-{}".format(os.environ['MIRROR_CHECK_SERVICE'])
  20 HEALTH_FILE = os.path.join(OUTPUT_DIR, "health")
  21 URL = os.environ['MIRROR_CHECK_URL']
  22 HEALTH_CHECK_URL = os.environ['MIRROR_CHECK_HEALTH_URL']
  23 INTERVAL = int(os.environ.get('MIRROR_CHECK_INTERVAL', '60'))
  24
  25 def retrieve_from_host(host, url):
  26     orig_create_connection = urllib3.util.connection.create_connection
  27     def patched_create_connection(address, *args, **kwargs):
  28         _host, port = address
  29         return orig_create_connection((host, port), *args, **kwargs)
  30     headers = {'User-Agent': 'mirror-health'}
  31     urllib3.util.connection.create_connection = patched_create_connection
  32     try:
  33         return requests.get(url, headers=headers, timeout=5, allow_redirects=False)
  34     finally:
  35         urllib3.util.connection.create_connection = orig_create_connection
  36
  37 def last_modified(response):
  38     lm = 0
  39     if response.status_code == 200 and response.headers.get('last-modified'):
  40         lm = calendar.timegm(parsedate(response.headers['last-modified']))
  41     return lm
  42
  43 def healthy(response):
  44     if response.status_code == 200:
  45         return True
  46     return False
  47
  48 def check_shutdown():
  49     if os.path.exists('/run/systemd/shutdown/scheduled'):
  50         logging.info("considering myself unhealthy, shutdown scheduled")
  51         return False
  52     return True
  53
  54 def check_uptodate():
  55     latest_ts = 0
  56     for host in HOSTS:
  57         try:
  58             lm = last_modified(retrieve_from_host(host, URL))
  59             logging.debug("lm for host %s: %s", host, lm)
  60             if healthy(retrieve_from_host(host, HEALTH_CHECK_URL)):
  61                 latest_ts = max(latest_ts, lm)
  62         except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
  63             pass
  64     try:
  65         local_lm = last_modified(retrieve_from_host('localhost', URL))
  66     except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
  67         return False
  68     logging.debug("lm for localhost: %s", local_lm)
  69     if local_lm < latest_ts:
  70         logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts)
  71         return False
  72     return True
  73
  74 while True:
  75     start = time.time()
  76     if check_shutdown() and check_uptodate():
  77         logging.info("considering myself healthy")
  78         open(HEALTH_FILE, 'w').write("OK")
  79     else:
  80         try:
  81             os.remove(HEALTH_FILE)
  82         except OSError:
  83             pass
  84     sleep_time = start + INTERVAL - time.time()
  85     logging.debug("sleeping for %d seconds", sleep_time)
  86     time.sleep(sleep_time)