X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=dsa-nagios-checks%2Fchecks%2Fdsa-check-backuppg;h=e8a859237f019a61126920b8b8371ca30b0df4d4;hb=b24070726f009d22a270de5b56cd484ab6154887;hp=92ce6cb6731bfdbb08ecc843ee3f8efbfad8b74c;hpb=f060cd469b7ae31ae606acb249210a9399fdbefe;p=mirror%2Fdsa-nagios.git

diff --git a/dsa-nagios-checks/checks/dsa-check-backuppg b/dsa-nagios-checks/checks/dsa-check-backuppg
index 92ce6cb..e8a8592 100755
--- a/dsa-nagios-checks/checks/dsa-check-backuppg
+++ b/dsa-nagios-checks/checks/dsa-check-backuppg
@@ -47,9 +47,11 @@ import copy
 import time
 import re
 import os
+import errno
 import sys
 import yaml
 import optparse
+import socket
 
 def load_conf(cf):
     if cf is not None:
@@ -65,6 +67,7 @@ def load_conf(cf):
     return config
 
 
+notices_seq = []
 problems_seq = []
 problems_per_db = {}
 global_expires = []
@@ -75,6 +78,12 @@ global_expires = []
 #    global problems_per_db
 #    if not host in problems_per_db: problems_per_db[host] = {}
 #    problems_per_db[host][db] = True
+def note_info(key, value, pre=None):
+    global notices_seq
+    if pre is None:
+        notices_seq.append("%s: %s"%(key, value))
+    else:
+        notices_seq.append("[%s] %s: %s"%(pre, key, value))
 
 def note_warning(key, value, pre=None):
     global problems_seq
@@ -89,11 +98,14 @@ def note_warning_db(host, db, key, value):
     problems_per_db[host][db] = True
 
 
-def wal_pre(w):
+def wal_pre(w, host, db):
     (w1,w2) = w
     if w2 == 0:
         w1 -= 1
-        w2 = 0xFE
+        if (host,db) in ( ('moszumanska', 'main'), ):
+            w2 = 0xFE
+        else:
+            w2 = 0xFF
     else:
         w2 -= 1
 
@@ -131,8 +143,23 @@ config = load_conf(options.conffile)
 
 os.chdir(config['rootdir'])
 for dir in os.listdir('.'):
+    if dir.startswith('.') or dir.endswith('.old'):
+        note_info('IGNORED', dir)
+        continue
+
     if not os.path.isdir(dir):
-        note_warning('NOT-A-DIR', dir)
+        try:
+            mtime = os.path.getmtime(dir)
+            ctime = os.path.getctime(dir)
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                continue
+            else:
+                raise e
+        if min(mtime, ctime) + 3600*4 > time.time():
+            note_info('IGNORED', dir)
+        else:
+            note_warning('NOT-A-DIR', dir)
         continue
 
     if not dir in config['backups']:
@@ -168,7 +195,7 @@ for dir in os.listdir('.'):
             continue
 
         (db, type) = r.groups(1)
-        if not isinstance(config['backups'][dir], list) or not db in config['backups'][dir]:
+        if not isinstance(config['backups'][dir], dict) or not db in config['backups'][dir]:
             if not db in ignored_dbs:
                 note_warning_db(dir, db, 'NOT-CONFIGURED', '%s/%s'%(dir, db))
             ignored_dbs[db] = True
@@ -184,6 +211,10 @@ for dir in os.listdir('.'):
             # can_expire_next: Can expire all files that we handle from now on
             backup_state[db]['can_expire_next'] = False
             backup_state[db]['expires'] = []
+            if isinstance(config['backups'][dir][db], dict) and 'timeline' in config['backups'][dir][db]:
+                backup_state[db]['timeline'] = config['backups'][dir][db]['timeline']
+            else:
+                backup_state[db]['timeline'] = 1
 
         # Apparently we already have seen a base backup and all its wal files
         # which we want to keep, so everything what we see now is older than
@@ -205,8 +236,16 @@ for dir in os.listdir('.'):
                 basefn = '%s.BASE.%s-%s.tar.gz'%(db, info['label'], info['start wal location'].split(' ',2)[0].replace('/', '_'))
                 baseffn = os.path.join(dir, basefn)
                 if not basefn in files:
-                    note_warning_db(dir, db, 'MISSING-BASE', basefn)
-                    continue
+                    basefn = '%s.BASE.%s.tar.gz'%(db, info['label'])
+                    baseffn = os.path.join(dir, basefn)
+                    if not basefn in files:
+                        m = re.match('([a-z0-9.]+)-\d{8}-\d{6}', info['label'])
+                        if m and (m.group(1) != socket.getfqdn()):
+                            note_info(dir, 'IGNORED-OTHER-BASE: '+basefn)
+                            continue
+                        else:
+                            note_warning_db(dir, db, 'MISSING-BASE', basefn)
+                            continue
                 files.remove(basefn)
                 if backup_state[db]['can_expire_next']:
                     backup_state[db]['expires'].append(baseffn)
@@ -238,8 +277,8 @@ for dir in os.listdir('.'):
                             backup_state[db]['can_expire_next'] = True
 
                 (timeline, wal1, wal2) = map(lambda x: int(x,16), r.groups())
-                if not timeline == 1:
-                    note_warning_db(dir, db, 'CANNOT-HANDLE-TIMELINES_NOT_1', ffn)
+                if not timeline == backup_state[db]['timeline']:
+                    note_warning_db(dir, db, 'UNEXPECTED-TIMELINE', ffn)
                     continue
 
                 thissegment = (wal1, wal2)
@@ -247,7 +286,7 @@ for dir in os.listdir('.'):
                     backup_state[db]['newest-wal'] = thissegment
                     backup_state[db]['newest-wal-file'] = ffn
                 else:
-                    if not wal_pre(backup_state[db]['oldest-wal']) == thissegment:
+                    if not wal_pre(backup_state[db]['oldest-wal'], dir, db) == thissegment:
                         note_warning_db(dir, db, 'WAL-MISSING-AFTER', ffn)
                         ignored_dbs[db] = True
                         continue
@@ -302,6 +341,8 @@ for dir in os.listdir('.'):
 
 for p in problems_seq:
     print p
+for p in notices_seq:
+    print p
 
 if options.expire:
     for f in global_expires: