X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=modules%2Froles%2Ffiles%2Fmirror_health%2Fmirror-health;h=d3df95f37bf2fa06258eae14533316b747fa68ca;hb=3d8cddc478904d0b4b8d58f99bd2c3cca23bdc42;hp=fb8241ba00416667baebc0077436d9912fdd2ed5;hpb=1552612184671215e066d620536e0594a2bb4ea5;p=mirror%2Fdsa-puppet.git

diff --git a/modules/roles/files/mirror_health/mirror-health b/modules/roles/files/mirror_health/mirror-health
index fb8241ba0..d3df95f37 100755
--- a/modules/roles/files/mirror_health/mirror-health
+++ b/modules/roles/files/mirror_health/mirror-health
@@ -5,6 +5,7 @@ import requests
 import time
 import calendar
 import logging
+import subprocess
 from email.utils import parsedate
 logging.basicConfig(level=logging.INFO)
 
@@ -14,39 +15,53 @@ HEALTH_FILE = os.path.join(OUTPUT_DIR, "health")
 URL = os.environ['MIRROR_CHECK_URL']
 INTERVAL = int(os.environ.get('MIRROR_CHECK_INTERVAL', '60'))
 
-latest_tz = 0
+latest_ts = 0
 
 def retrieve_from_host(host, url):
     proxies = {
         'http': 'http://{}:80'.format(host),
         'https': 'http://{}:443'.format(host),
     }
-    return requests.get(url, timeout=5, proxies=proxies)
+    return requests.get(url, timeout=5, proxies=proxies, allow_redirects=False)
 
 def last_modified(response):
     lm = 0
-    if response.status_code == '200' and response.headers.get('last-modified'):
+    if response.status_code == 200 and response.headers.get('last-modified'):
         lm = calendar.timegm(parsedate(response.headers['last-modified']))
     return lm
 
-while True:
-    start = time.time()
+def check_shutdown():
+    if subprocess.call(['dsa-is-shutdown-scheduled']) == 0:
+        logging.info("considering myself unhealthy, shutdown scheduled")
+        return False
+    return True
+
+def check_uptodate():
+    global latest_ts
     for host in HOSTS:
-        lm = last_modified(retrieve_from_host(host, URL))
-        logging.debug("lm for host %s: %s", host, lm)
-        if lm > latest_tz:
-            latest_tz = lm
+        try:
+            lm = last_modified(retrieve_from_host(host, URL))
+            logging.debug("lm for host %s: %s", host, lm)
+            latest_ts = max(latest_ts, lm)
+        except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout):
+            pass
     local_lm = last_modified(retrieve_from_host('localhost', URL))
-    logging.debug("lm for localhost: %s", lm)
-    if latest_tz <= local_lm:
+    logging.debug("lm for localhost: %s", local_lm)
+    if local_lm < latest_ts:
+        logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts)
+        return False
+    return True
+
+while True:
+    start = time.time()
+    if check_shutdown() and check_uptodate():
+        logging.info("considering myself healthy")
+        open(HEALTH_FILE, 'w').write("OK")
+    else:
         try:
-            logging.debug("considering myself unhealthy")
             os.remove(HEALTH_FILE)
         except OSError:
             pass
-    else:
-        logging.debug("considering myself healthy")
-        open(HEALTH_FILE, 'w').write("OK")
     sleep_time = start + INTERVAL - time.time()
     logging.debug("sleeping for %d seconds", sleep_time)
     time.sleep(sleep_time)