X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=modules%2Froles%2Ffiles%2Fmirror_health%2Fmirror-health;h=57703668e387bc197a72b1462568681f0a0689ba;hb=e2e0d9005a132769cff685844de0c063902037ba;hp=62148828f93737d2bab62286f7ed82d28d1142c4;hpb=b778bf45ae54ad750c167847ee8bac0182d7a9ed;p=mirror%2Fdsa-puppet.git

diff --git a/modules/roles/files/mirror_health/mirror-health b/modules/roles/files/mirror_health/mirror-health
index 62148828f..57703668e 100755
--- a/modules/roles/files/mirror_health/mirror-health
+++ b/modules/roles/files/mirror_health/mirror-health
@@ -5,6 +5,7 @@ import requests
 import time
 import calendar
 import logging
+import subprocess
 from email.utils import parsedate
 logging.basicConfig(level=logging.INFO)
 
@@ -29,24 +30,41 @@ def last_modified(response):
         lm = calendar.timegm(parsedate(response.headers['last-modified']))
     return lm
 
-while True:
-    start = time.time()
+def check_shutdown():
+    if subprocess.call(['dsa-is-shutdown-scheduled']) == 0:
+        logging.info("considering myself unhealthy, shutdown scheduled")
+        return False
+    return True
+
+def check_uptodate():
+    global latest_ts
     for host in HOSTS:
-        lm = last_modified(retrieve_from_host(host, URL))
-        logging.debug("lm for host %s: %s", host, lm)
-        if lm > latest_ts:
-            latest_ts = lm
-    local_lm = last_modified(retrieve_from_host('localhost', URL))
+        try:
+            lm = last_modified(retrieve_from_host(host, URL))
+            logging.debug("lm for host %s: %s", host, lm)
+            latest_ts = max(latest_ts, lm)
+        except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout):
+            pass
+    try:
+        local_lm = last_modified(retrieve_from_host('localhost', URL))
+    except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout):
+        return False
     logging.debug("lm for localhost: %s", local_lm)
     if local_lm < latest_ts:
+        logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts)
+        return False
+    return True
+
+while True:
+    start = time.time()
+    if check_shutdown() and check_uptodate():
+        logging.info("considering myself healthy")
+        open(HEALTH_FILE, 'w').write("OK")
+    else:
         try:
-            logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts)
             os.remove(HEALTH_FILE)
         except OSError:
             pass
-    else:
-        logging.info("considering myself healthy")
-        open(HEALTH_FILE, 'w').write("OK")
     sleep_time = start + INTERVAL - time.time()
     logging.debug("sleeping for %d seconds", sleep_time)
     time.sleep(sleep_time)