import time
import calendar
import logging
+import subprocess
from email.utils import parsedate
logging.basicConfig(level=logging.INFO)
OUTPUT_DIR = "/run/dsa-mirror-health-{}".format(os.environ['MIRROR_CHECK_SERVICE'])
HEALTH_FILE = os.path.join(OUTPUT_DIR, "health")
URL = os.environ['MIRROR_CHECK_URL']
+HEALTH_CHECK_URL = os.environ['MIRROR_CHECK_HEALTH_URL']
INTERVAL = int(os.environ.get('MIRROR_CHECK_INTERVAL', '60'))
-latest_ts = 0
-
def retrieve_from_host(host, url):
proxies = {
'http': 'http://{}:80'.format(host),
lm = calendar.timegm(parsedate(response.headers['last-modified']))
return lm
+def healthy(response):
+ if response.status_code == 200:
+ return True
+ return False
+
+def check_shutdown():
+ if subprocess.call(['dsa-is-shutdown-scheduled']) == 0:
+ logging.info("considering myself unhealthy, shutdown scheduled")
+ return False
+ return True
+
+def check_uptodate():
+ latest_ts = 0
+ for host in HOSTS:
+ try:
+ lm = last_modified(retrieve_from_host(host, URL))
+ logging.debug("lm for host %s: %s", host, lm)
+ if healthy(retrieve_from_host(host, HEALTH_CHECK_URL)):
+ latest_ts = max(latest_ts, lm)
+ except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
+ pass
+ try:
+ local_lm = last_modified(retrieve_from_host('localhost', URL))
+ except (requests.exceptions.ProxyError, requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
+ return False
+ logging.debug("lm for localhost: %s", local_lm)
+ if local_lm < latest_ts:
+ logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts)
+ return False
+ return True
+
while True:
start = time.time()
- for host in HOSTS:
- lm = last_modified(retrieve_from_host(host, URL))
- logging.debug("lm for host %s: %s", host, lm)
- if lm > latest_ts:
- latest_ts = lm
- local_lm = last_modified(retrieve_from_host('localhost', URL))
- logging.debug("lm for localhost: %s", lm)
- if latest_ts <= local_lm:
+ if check_shutdown() and check_uptodate():
+ logging.info("considering myself healthy")
+ open(HEALTH_FILE, 'w').write("OK")
+ else:
try:
- logging.info("considering myself unhealthy my ts=%s latest_ts=%s", local_lm, latest_ts)
os.remove(HEALTH_FILE)
except OSError:
pass
- else:
- logging.info("considering myself healthy")
- open(HEALTH_FILE, 'w').write("OK")
sleep_time = start + INTERVAL - time.time()
logging.debug("sleeping for %d seconds", sleep_time)
time.sleep(sleep_time)