## vim:set et ts=2 sw=2 ai:
# Send email reminders to users having sizable homedirs.
##
-# Copyright (c) 2013 Philipp Kern <phil@philkern.de>
-# Copyright (c) 2013 Peter Palfrader <peter@palfrader.org>
+# Copyright (c) 2013 Philipp Kern <pkern@debian.org>
+# Copyright (c) 2013, 2014 Peter Palfrader <peter@palfrader.org>
# Copyright (c) 2013 Luca Filipozzi <lfilipoz@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
-from __future__ import print_function
-
from collections import defaultdict
+from dsa_mq.connection import Connection
+from dsa_mq.config import Config
import email
import email.mime.text
import glob
import logging
+from optparse import OptionParser
import os.path
import platform
import pwd
-import random
import subprocess
import struct
import time
# avoid base64 encoding for utf-8
email.charset.add_charset('utf-8', email.charset.SHORTEST, email.charset.QP)
-SENDMAIL = ['/usr/sbin/sendmail', '-t', '-oi']
-#SENDMAIL = ['/bin/cat']
-
-EXPLANATIONS = [
-u"""\
-{hostname}'s /home is, unfortunately, not infinite in size. If you have
-anything in there that you no longer need, please clean it up."""
-,u"""\
-Can you please look at your $HOME on {hostname} and remove files which
-you no longer need (such as old sources)."""
-,u"""\
-Thanks for your porting effort on {hostname}!
+parser = OptionParser()
+parser.add_option("-D", "--dryrun",
+ action="store_true", default=False,
+ help="Dry run mode")
-Please note that on most porterboxes /home is quite small, so please remove
-files that you do not need anymore."""
- ]
+parser.add_option("-d", "--debug",
+ action="store_true", default=False,
+ help="Enable debug output")
+
+(options, args) = parser.parse_args()
+options.section = 'dsa-homedirs'
+options.config = '/etc/dsa/pubsub.conf'
+if os.access(options.config, os.R_OK):
+ mq_config = Config(options)
+ mq_conf = {
+ 'rabbit_userid': mq_config.username,
+ 'rabbit_password': mq_config.password,
+ 'rabbit_virtual_host': mq_config.vhost,
+ 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
+ 'use_ssl': False
+ }
+else:
+ mq_config = None
+
+if options.dryrun:
+ SENDMAIL_COMMAND = ['/bin/cat']
+ RM_COMMAND = ['/bin/echo', 'Would remove']
+else:
+ SENDMAIL_COMMAND = ['/usr/sbin/sendmail', '-t', '-oi']
+ RM_COMMAND = ['/bin/rm', '-rf']
CRITERIA = [
- { 'days': 5, 'size': 10240 },
- { 'days': 10, 'size': 1024 },
- { 'days': 30, 'size': 100 },
- { 'days': 60, 'size': 60 },
- { 'days': 90, 'size': 10 }
+ { 'size': 10240, 'notifyafter': 5, 'deleteafter': 40 },
+ { 'size': 1024, 'notifyafter': 10, 'deleteafter': 50 },
+ { 'size': 100, 'notifyafter': 30, 'deleteafter': 90 },
+ { 'size': 20, 'notifyafter': 90, 'deleteafter': 150 },
+ { 'size': 5, 'deleteafter': 700 }
]
-EXCLUDED_USERNAMES = ['lost+found']
+EXCLUDED_USERNAMES = ['lost+found', 'debian', 'buildd', 'd-i']
MAIL_FROM = 'debian-admin (via Cron) <bulk@admin.debian.org>'
MAIL_TO = '{username}@{hostname}.debian.org'
MAIL_CC = 'debian-admin (bulk sink) <bulk@admin.debian.org>'
MAIL_REPLYTO = 'debian-admin <dsa@debian.org>'
MAIL_SUBJECT = 'Please clean up ~{username} on {hostname}.debian.org'
MAIL_MESSAGE = u"""\
-Hi {name}!
+Hi {realname}!
-{explanation}
+Thanks for your porting effort on {hostname}!
+
+Please note that, on most porterboxes, /home is quite small, so please
+remove files that you do not need anymore.
For your information, you last logged into {hostname} {days_ago} days
ago, and your home directory there is {homedir_size} MB in size.
Debian System Administration Team via Cron
-PS: replies not required.
+PS: A reply is not required.
"""
class Error(Exception):
pass
class LastlogTimes(dict):
- LASTLOG_STRUCT = '=L32s256s'
-
+ LASTLOG_STRUCT_32 = '=L32s256s'
+ LASTLOG_STRUCT_64 = '=Q32s256s'
+
def __init__(self):
- record_size = struct.calcsize(self.LASTLOG_STRUCT)
+ record_size_32 = struct.calcsize(self.LASTLOG_STRUCT_32)
+ record_size_64 = struct.calcsize(self.LASTLOG_STRUCT_64)
+ # some 64bit arches have 32bit-compatible lastlog structures, others don't,
+ # in apparently incoherent ways, so hardcode a list...
+ if platform.machine() in ('aarch64', 's390x'):
+ self.LASTLOG_STRUCT = self.LASTLOG_STRUCT_64
+ record_size = record_size_64
+ else:
+ self.LASTLOG_STRUCT = self.LASTLOG_STRUCT_32
+ record_size = record_size_32
with open('/var/log/lastlog', 'r') as fp:
uid = -1 # there is one record per uid in lastlog
for record in iter(lambda: fp.read(record_size), ''):
+ if len(record) != record_size:
+ raise RuntimeError('lastlog has unexpected size, read %d instead of %d'
+ % (len(record), record_size))
uid += 1 # so keep incrementing uid for each record read
lastlog_time, _, _ = list(struct.unpack(self.LASTLOG_STRUCT, record))
- if lastlog_time == 0:
- continue
+ if lastlog_time < 0:
+ raise RuntimeError('unexpected last login time %d for user %s'
+ % (lastlog_time, pwd.getpwuid(uid).pw_name))
try:
self[pwd.getpwuid(uid).pw_name] = lastlog_time
except KeyError:
- logging.error('could not resolve username from uid')
+ # this is a normal condition
continue
class HomedirSizes(dict):
def __init__(self):
for direntry in glob.glob('/home/*'):
username = os.path.basename(direntry)
+
if username in EXCLUDED_USERNAMES:
continue
+
try:
pwinfo = pwd.getpwnam(username)
except KeyError:
if os.path.isdir(direntry):
logging.warning('directory %s exists on %s but there is no %s user', direntry, platform.node(), username)
continue
+
+ if pwinfo.pw_dir != direntry:
+ logging.warning('home directory for %s is not %s, but that exists. confused.', username, direntry)
+ continue
+
command = ['/usr/bin/du', '-ms', pwinfo.pw_dir]
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
self.lastlog_times = LastlogTimes()
self.homedir_sizes = HomedirSizes()
- def send_mail(self, **kwargs):
+ def notify(self, **kwargs):
msg = email.mime.text.MIMEText(MAIL_MESSAGE.format(**kwargs), _charset='UTF-8')
msg['From'] = MAIL_FROM.format(**kwargs)
msg['To'] = MAIL_TO.format(**kwargs)
msg['Subject'] = MAIL_SUBJECT.format(**kwargs)
msg['Precedence'] = "bulk"
msg['Auto-Submitted'] = "auto-generated by mail-big-homedirs"
- p = subprocess.Popen(SENDMAIL, stdin=subprocess.PIPE)
+ p = subprocess.Popen(SENDMAIL_COMMAND, stdin=subprocess.PIPE)
p.communicate(msg.as_string())
logging.debug(msg.as_string())
if p.returncode != 0:
raise SendmailError
+ def remove(self, **kwargs):
+ try:
+ pwinfo = pwd.getpwnam(kwargs.get('username'))
+ except KeyError:
+ return
+
+ command = RM_COMMAND + [pwinfo.pw_dir]
+ p = subprocess.check_call(command)
+
def run(self):
current_time = time.time()
+ conn = None
+ try:
+ data = {}
+ for user in set(self.homedir_sizes.keys()) | \
+ set(self.lastlog_times.keys()):
+ data[user] = {
+ 'homedir': self.homedir_sizes.get(user, 0),
+ 'lastlog': self.lastlog_times.get(user, 0),
+ }
+
+ if mq_config is not None:
+ msg = {
+ 'timestamp': current_time,
+ 'data': data,
+ 'host': platform.node(),
+ }
+ conn = Connection(conf=mq_conf)
+ conn.topic_send(mq_config.topic,
+ msg,
+ exchange_name=mq_config.exchange,
+ timeout=5)
+ except Exception, e:
+ logging.error("Error sending: %s" % e)
+ finally:
+ if conn:
+ conn.close()
+
for username, homedir_size in self.homedir_sizes.iteritems():
try:
- name = pwd.getpwnam(username).pw_gecos.decode('utf-8').split(',', 1)[0].split(' ', 1)[0]
+ realname = pwd.getpwnam(username).pw_gecos.decode('utf-8').split(',', 1)[0]
except:
- name = username
- lastlog_time = self.lastlog_times[username]
+ realname = username
+ lastlog_time = self.lastlog_times.get(username, 0)
days_ago = int( (current_time - lastlog_time) / 3600 / 24 )
- if [x for x in CRITERIA if days_ago >= x['days'] and homedir_size >= x['size']]:
- explanation = EXPLANATIONS[random.randint(0,len(EXPLANATIONS)-1)].format(hostname=platform.node())
- self.send_mail(hostname=platform.node(), username=username, name=name, explanation=explanation, homedir_size=homedir_size, days_ago=days_ago)
+ kwargs = {
+ 'hostname': platform.node(),
+ 'username': username,
+ 'realname': realname,
+ 'homedir_size': homedir_size,
+ 'days_ago': days_ago
+ }
+
+ notify = False
+ remove = False
+ for x in CRITERIA:
+ if homedir_size > x['size'] and 'notifyafter' in x and days_ago >= x['notifyafter']:
+ notify = True
+ if homedir_size > x['size'] and 'deleteafter' in x and days_ago >= x['deleteafter']:
+ remove = True
+
+ if remove:
+ self.remove(**kwargs)
+ elif notify:
+ self.notify(**kwargs)
if __name__ == '__main__':
- logging.basicConfig()
- # DEBUG for debugging, ERROR for production.
- logging.getLogger().setLevel(logging.ERROR)
+ lvl = logging.ERROR
+ if options.debug:
+ lvl = logging.DEBUG
+ logging.basicConfig(level=lvl)
HomedirReminder().run()