X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=modules%2Fporterbox%2Ffiles%2Fmail-big-homedirs;h=d5277aef41e66ca190c65f973d86120b447f2015;hb=ca7c18007c9f3f380dbf71841e97204eb90bb4ca;hp=c211619a46618b83d647dd0a538bd34e4b7ad79b;hpb=dea39c11d4df6abb11c89f97b9988a664a0d1c3b;p=mirror%2Fdsa-puppet.git diff --git a/modules/porterbox/files/mail-big-homedirs b/modules/porterbox/files/mail-big-homedirs index c211619a4..d5277aef4 100755 --- a/modules/porterbox/files/mail-big-homedirs +++ b/modules/porterbox/files/mail-big-homedirs @@ -1,9 +1,9 @@ #!/usr/bin/python ## vim:set et ts=2 sw=2 ai: -# homedir_reminder.py - Reminds users about sizable homedirs. +# Send email reminders to users having sizable homedirs. ## -# Copyright (c) 2013 Philipp Kern -# Copyright (c) 2013 Peter Palfrader +# Copyright (c) 2013 Philipp Kern +# Copyright (c) 2013, 2014 Peter Palfrader # Copyright (c) 2013 Luca Filipozzi # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -24,13 +24,14 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from __future__ import print_function - from collections import defaultdict +from dsa_mq.connection import Connection +from dsa_mq.config import Config import email import email.mime.text import glob import logging +from optparse import OptionParser import os.path import platform import pwd @@ -38,44 +39,72 @@ import subprocess import struct import time import StringIO -import pwd # avoid base64 encoding for utf-8 email.charset.add_charset('utf-8', email.charset.SHORTEST, email.charset.QP) - -SENDMAIL = ['/usr/sbin/sendmail', '-t', '-oi'] -#SENDMAIL = ['/bin/cat'] - -REPORT_SIZES = [ - { 'days': 5, 'size': 10240 }, - { 'days': 10, 'size': 1024 }, - { 'days': 30, 'size': 100 }, - { 'days': 75, 'size': 10 } +parser = OptionParser() +parser.add_option("-D", "--dryrun", + action="store_true", default=False, + help="Dry run mode") + +parser.add_option("-d", "--debug", + action="store_true", default=False, + help="Enable debug output") + +(options, args) = parser.parse_args() +options.section = 'dsa-homedirs' +options.config = '/etc/dsa/pubsub.conf' +config = Config(options) +mq_conf = { + 'rabbit_userid': config.username, + 'rabbit_password': config.password, + 'rabbit_virtual_host': config.vhost, + 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'], + 'use_ssl': False +} + +if options.dryrun: + SENDMAIL_COMMAND = ['/bin/cat'] + RM_COMMAND = ['/bin/echo', 'Would remove'] +else: + SENDMAIL_COMMAND = ['/usr/sbin/sendmail', '-t', '-oi'] + RM_COMMAND = ['/bin/rm', '-rf'] + +CRITERIA = [ + { 'size': 10240, 'notifyafter': 5, 'deleteafter': 40 }, + { 'size': 1024, 'notifyafter': 10, 'deleteafter': 50 }, + { 'size': 100, 'notifyafter': 30, 'deleteafter': 90 }, + { 'size': 20, 'notifyafter': 90, 'deleteafter': 150 }, + { 'size': 5, 'deleteafter': 700 } ] -USER_EXCLUSION_LIST = ['lost+found'] +EXCLUDED_USERNAMES = ['lost+found', 'debian'] MAIL_FROM = 'debian-admin (via Cron) ' MAIL_TO = '{username}@{hostname}.debian.org' MAIL_CC = 'debian-admin (bulk sink) ' MAIL_REPLYTO = 'debian-admin ' MAIL_SUBJECT = 'Please clean up ~{username} on {hostname}.debian.org' -MAIL_TEXT = u"""\ -Hi {name}, +MAIL_MESSAGE = u"""\ +Hi {realname}! -you last logged into {hostname} {days_ago} days ago, and your home -directory there is {homedir_size}MB in size. +Thanks for your porting effort on {hostname}! -Disk space on porter boxes is often limited. Please assist us by freeing up -space by deleting schroots and source/build trees in ~{username} as soon as -feasible. +Please note that, on most porterboxes, /home is quite small, so please +remove files that you do not need anymore. -If you currently do not use {hostname}, please keep ~{username} under 10MB. +For your information, you last logged into {hostname} {days_ago} days +ago, and your home directory there is {homedir_size} MB in size. + +If you currently do not use {hostname}, please keep ~{username} under +10 MB, if possible. + +Please assist us in freeing up space by deleting schroots, also. Thanks, Debian System Administration Team via Cron -PS: replies not required. +PS: A reply is not required. """ class Error(Exception): @@ -84,116 +113,143 @@ class Error(Exception): class SendmailError(Error): pass -class LastLog(object): +class LastlogTimes(dict): LASTLOG_STRUCT = '=L32s256s' - def __init__(self, fname='/var/log/lastlog'): + def __init__(self): record_size = struct.calcsize(self.LASTLOG_STRUCT) - self.records = {} - with open(fname, 'r') as fp: - uid = -1 + with open('/var/log/lastlog', 'r') as fp: + uid = -1 # there is one record per uid in lastlog for record in iter(lambda: fp.read(record_size), ''): - uid += 1 - last_login, _, _ = list(struct.unpack(self.LASTLOG_STRUCT, record)) - if last_login == 0: - continue + uid += 1 # so keep incrementing uid for each record read + lastlog_time, _, _ = list(struct.unpack(self.LASTLOG_STRUCT, record)) try: - self.records[pwd.getpwuid(uid).pw_name] = last_login + self[pwd.getpwuid(uid).pw_name] = lastlog_time except KeyError: + # this is a normal condition continue - def last_login_for_user(self, username): - return self.records.get(username, 0) - -class HomedirReminder(object): +class HomedirSizes(dict): def __init__(self): - self.lastlog = LastLog() - self.generate_homedir_list() - - def parse_utmp(self): - self.utmp_records = defaultdict(list) - for wtmpfile in glob.glob('/var/log/wtmp*'): - for entry in utmp.UtmpRecord(wtmpfile): - # TODO: Login, does not account for non-idle sessions. - self.utmp_records[entry.ut_user].append(entry.ut_tv[0]) - for username, timestamps in self.utmp_records.iteritems(): - self.utmp_records[username] = sorted(timestamps)[-1] - - def last_login_for_user(self, username): - return self.lastlog.last_login_for_user(username) - - def generate_homedir_list(self): - self.homedir_sizes = {} for direntry in glob.glob('/home/*'): username = os.path.basename(direntry) + + if username in EXCLUDED_USERNAMES: + continue + try: pwinfo = pwd.getpwnam(username) except KeyError: if os.path.isdir(direntry): - logging.warning('Directory %s exists on %s but there is no %s user', direntry, platform.node(), username) + logging.warning('directory %s exists on %s but there is no %s user', direntry, platform.node(), username) continue - homedir = pwinfo.pw_dir - if username in USER_EXCLUSION_LIST: + if pwinfo.pw_dir != direntry: + logging.warning('home directory for %s is not %s, but that exists. confused.', username, direntry) continue - # Ignore errors from du. - command = ['/usr/bin/du', '-ms', homedir] - p = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + + command = ['/usr/bin/du', '-ms', pwinfo.pw_dir] + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() - if p.returncode != 0: + if p.returncode != 0: # ignore errors from du logging.info('%s failed:', ' '.join(command)) logging.info(stderr) + continue try: - size = int(stdout.split('\t')[0]) + self[username] = int(stdout.split('\t')[0]) except ValueError: - logging.error('Could not convert size output from %s: %s', - ' '.join(command), stdout) + logging.error('could not convert size output from %s: %s', ' '.join(command), stdout) continue - self.homedir_sizes[username] = size - def send_mail(self, **kwargs): - msg = email.mime.text.MIMEText(MAIL_TEXT.format(**kwargs), _charset='UTF-8') +class HomedirReminder(object): + def __init__(self): + self.lastlog_times = LastlogTimes() + self.homedir_sizes = HomedirSizes() + + def notify(self, **kwargs): + msg = email.mime.text.MIMEText(MAIL_MESSAGE.format(**kwargs), _charset='UTF-8') msg['From'] = MAIL_FROM.format(**kwargs) msg['To'] = MAIL_TO.format(**kwargs) - if MAIL_CC != "": msg['Cc'] = MAIL_CC.format(**kwargs) - if MAIL_REPLYTO != "": msg['Reply-To'] = MAIL_REPLYTO.format(**kwargs) + if MAIL_CC != "": + msg['Cc'] = MAIL_CC.format(**kwargs) + if MAIL_REPLYTO != "": + msg['Reply-To'] = MAIL_REPLYTO.format(**kwargs) msg['Subject'] = MAIL_SUBJECT.format(**kwargs) msg['Precedence'] = "bulk" msg['Auto-Submitted'] = "auto-generated by mail-big-homedirs" - p = subprocess.Popen(SENDMAIL, stdin=subprocess.PIPE) + p = subprocess.Popen(SENDMAIL_COMMAND, stdin=subprocess.PIPE) p.communicate(msg.as_string()) logging.debug(msg.as_string()) if p.returncode != 0: raise SendmailError + def remove(self, **kwargs): + try: + pwinfo = pwd.getpwnam(kwargs.get('username')) + except KeyError: + return + + command = RM_COMMAND + [pwinfo.pw_dir] + p = subprocess.check_call(command) + def run(self): current_time = time.time() - for username, homedir_size in self.homedir_sizes.iteritems(): - last_login = self.last_login_for_user(username) - logging.info('user %s: size %dMB, last login: %d', username, homedir_size, last_login) - days_ago = int( (current_time - last_login) / 3600 / 24 ) - - reportsize = None - for e in REPORT_SIZES: - if days_ago > e['days']: reportsize = e['size'] + try: + data = {} + for user in set(self.homedir_sizes.keys()) + + set(self.lastlog_times.keys()): + data[user] = { + 'homedir': self.homedir_sizes.get(user, 0), + 'lastlog': self.lastlog_times.get(user, 0), + } + + msg = { + 'timestamp': current_time, + 'data': data, + 'host': platform.node(), + } + conn = Connection(conf=mq_conf) + conn.topic_send(config.topic, + msg, + exchange_name=config.exchange, + timeout=5) + except Exception, e: + LOG.error("Error sending: %s" % e) + finally: + if conn: + conn.close() - if reportsize is not None and homedir_size > reportsize: - logging.warning('Homedir of user %s is %d and did not login for a while', username, homedir_size) - try: - name = pwd.getpwnam(username).pw_gecos.decode('utf-8') - name = name.split(',', 1)[0] - except: - name = username - self.send_mail(hostname=platform.node(), - username=username, - name=name, - homedir_size=homedir_size, - days_ago=days_ago) + for username, homedir_size in self.homedir_sizes.iteritems(): + try: + realname = pwd.getpwnam(username).pw_gecos.decode('utf-8').split(',', 1)[0] + except: + realname = username + lastlog_time = self.lastlog_times[username] + days_ago = int( (current_time - lastlog_time) / 3600 / 24 ) + kwargs = { + 'hostname': platform.node(), + 'username': username, + 'realname': realname, + 'homedir_size': homedir_size, + 'days_ago': days_ago + } + + notify = False + remove = False + for x in CRITERIA: + if homedir_size > x['size'] and 'notifyafter' in x and days_ago >= x['notifyafter']: + notify = True + if homedir_size > x['size'] and 'deleteafter' in x and days_ago >= x['deleteafter']: + remove = True + + if remove: + self.remove(**kwargs) + elif notify: + self.notify(**kwargs) if __name__ == '__main__': - logging.basicConfig() - # DEBUG for debugging, ERROR for production. - logging.getLogger().setLevel(logging.ERROR) + lvl = logging.ERROR + if options.debug: + lvl = logging.DEBUG + logging.basicConfig(level=lvl) HomedirReminder().run()