X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=ud-generate;h=a787dfad5df514b5f7120ac6d5e92767e32d64c0;hb=HEAD;hp=1782f096b9eb93c75008b4799f53d11cbe93b22b;hpb=14908ffbe8886fdd85bf143ef14200e0171c18cd;p=mirror%2Fuserdir-ldap.git diff --git a/ud-generate b/ud-generate index 1782f09..a787dfa 100755 --- a/ud-generate +++ b/ud-generate @@ -6,9 +6,13 @@ # Copyright (c) 2003-2004 James Troup # Copyright (c) 2004-2005,7 Joey Schulze # Copyright (c) 2001-2007 Ryan Murray -# Copyright (c) 2008 Peter Palfrader +# Copyright (c) 2008,2009,2010,2011 Peter Palfrader # Copyright (c) 2008 Andreas Barth # Copyright (c) 2008 Mark Hymers +# Copyright (c) 2008 Luk Claes +# Copyright (c) 2008 Thomas Viehmann +# Copyright (c) 2009 Stephen Gran +# Copyright (c) 2010 Helmut Grohne # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,1073 +28,1486 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import string, re, time, ldap, getopt, sys, os, pwd, posix, socket, base64, sha, shutil, errno, tarfile, grp -from userdir_ldap import *; +from dsa_mq.connection import Connection +from dsa_mq.config import Config + +import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm +from userdir_ldap import * +from userdir_exceptions import * +import UDLdap +from xml.etree.ElementTree import Element, SubElement, Comment +from xml.etree import ElementTree +from xml.dom import minidom +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO +try: + import simplejson as json +except ImportError: + import json + if not '__author__' in json.__dict__: + sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n") + sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n") + +if os.getuid() == 0: + sys.stderr.write("You should probably not run ud-generate as root.\n") + sys.exit(1) + + +# +# GLOBAL STATE +# +GroupIDMap = None +SubGroupMap = None -global Allowed; -global CurrentHost; -PasswdAttrs = None; -GroupIDMap = {}; -Allowed = None; -CurrentHost = ""; UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' +MAX_UD_AGE = 3600*24 -EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)?$"); -BSMTPCheck = re.compile(".*mx 0 (gluck)\.debian\.org\..*",re.DOTALL); +EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$") +BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL) +PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]") +IsDebianHost = re.compile(ConfModule.dns_hostmatch) +isSSHFP = re.compile("^\s*IN\s+SSHFP") DNSZone = ".debian.net" Keyrings = ConfModule.sync_keyrings.split(":") +GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None) +GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None) +GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", ".")) +MX_remap = json.loads(ConfModule.MX_remap) +use_mq = getattr(ConfModule, "use_mq", True) + +rtc_realm = getattr(ConfModule, "rtc_realm", None) +rtc_append = getattr(ConfModule, "rtc_append", None) + +def prettify(elem): + """Return a pretty-printed XML string for the Element. + """ + rough_string = ElementTree.tostring(elem, 'utf-8') + reparsed = minidom.parseString(rough_string) + return reparsed.toprettyxml(indent=" ") def safe_makedirs(dir): - try: - os.makedirs(dir) - except OSError, e: - if e.errno == errno.EEXIST: - pass - else: - raise e + try: + os.makedirs(dir) + except OSError, e: + if e.errno == errno.EEXIST: + pass + else: + raise e def safe_rmtree(dir): - try: - shutil.rmtree(dir) - except OSError, e: - if e.errno == errno.ENOENT: - pass - else: - raise e + try: + shutil.rmtree(dir) + except OSError, e: + if e.errno == errno.ENOENT: + pass + else: + raise e + +def get_lock(fn, wait=5*60): + f = open(fn, "w") + sl = 0.1 + ends = time.time() + wait + + while True: + success = False + try: + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return f + except IOError: + pass + if time.time() >= ends: + return None + sl = min(sl*2, 10, ends - time.time()) + time.sleep(sl) + return None + def Sanitize(Str): - return Str.translate(string.maketrans("\n\r\t","$$$")) + return Str.translate(string.maketrans("\n\r\t", "$$$")) -def DoLink(From,To,File): - try: posix.remove(To+File); - except: pass; - posix.link(From+File,To+File); +def DoLink(From, To, File): + try: + posix.remove(To + File) + except: + pass + posix.link(From + File, To + File) -# See if this user is in the group list -def IsInGroup(DnRecord): - if Allowed == None: - return 1; +def IsRetired(account): + """ + Looks for accountStatus in the LDAP record and tries to + match it against one of the known retired statuses + """ + + status = account['accountStatus'] + + line = status.split() + status = line[0] + + if status == "inactive": + return True + + elif status == "memorial": + return True + elif status == "retiring": + # We'll give them a few extra days over what we said + age = 6 * 31 * 24 * 60 * 60 + try: + return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age + except IndexError: + return False + except ValueError: + return False + + return False + +# See if this user is in the group list +def IsInGroup(account, allowed, current_host): # See if the primary group is in the list - if Allowed.has_key(GetAttr(DnRecord,"gidNumber")) != 0: - return 1; + if str(account['gidNumber']) in allowed: return True # Check the host based ACL - if DnRecord[1].has_key("allowedHost") != 0: - for I in DnRecord[1]["allowedHost"]: - if CurrentHost == I: - return 1; + if account.is_allowed_by_hostacl(current_host): return True # See if there are supplementary groups - if DnRecord[1].has_key("supplementaryGid") == 0: - return 0; + if not 'supplementaryGid' in account: return False - # Check the supplementary groups - for I in DnRecord[1]["supplementaryGid"]: - if Allowed.has_key(I): - return 1; - return 0; + supgroups=[] + addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host) + for g in supgroups: + if g in allowed: + return True + return False -def Die(File,F,Fdb): +def Die(File, F, Fdb): if F != None: - F.close(); + F.close() if Fdb != None: - Fdb.close(); - try: os.remove(File + ".tmp"); - except: pass; - try: os.remove(File + ".tdb.tmp"); - except: pass; - -def Done(File,F,Fdb): - if F != None: - F.close(); - os.rename(File + ".tmp",File); - if Fdb != None: - Fdb.close(); - os.rename(File + ".tdb.tmp",File+".tdb"); - -# Generate the password list -def GenPasswd(l,File,HomePrefix,PwdMarker): - F = None; - try: - F = open(File + ".tdb.tmp","w"); + Fdb.close() + try: + os.remove(File + ".tmp") + except: + pass + try: + os.remove(File + ".tdb.tmp") + except: + pass + +def Done(File, F, Fdb): + if F != None: + F.close() + os.rename(File + ".tmp", File) + if Fdb != None: + Fdb.close() + os.rename(File + ".tdb.tmp", File + ".tdb") - userlist = {} - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - I = 0; - for x in PasswdAttrs: - if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0: - continue; - - # Do not let people try to buffer overflow some busted passwd parser. - if len(GetAttr(x,"gecos")) > 100 or len(GetAttr(x,"loginShell")) > 50: - continue; - - userlist[GetAttr(x, "uid")] = int(GetAttr(x, "gidNumber")) - Line = "%s:%s:%s:%s:%s:%s%s:%s" % (GetAttr(x,"uid"),\ - PwdMarker,\ - GetAttr(x,"uidNumber"),GetAttr(x,"gidNumber"),\ - GetAttr(x,"gecos"),HomePrefix,GetAttr(x,"uid"),\ - GetAttr(x,"loginShell")); - - Line = Sanitize(Line) + "\n"; - F.write("0%u %s" % (I,Line)); - F.write(".%s %s" % (GetAttr(x,"uid"),Line)); - F.write("=%s %s" % (GetAttr(x,"uidNumber"),Line)); - I = I + 1; - - # Oops, something unspeakable happened. - except: - Die(File,None,F); - raise; - Done(File,None,F); - - # Return the list of users so we know which keys to export - return userlist +# Generate the password list +def GenPasswd(accounts, File, HomePrefix, PwdMarker): + F = None + try: + F = open(File + ".tdb.tmp", "w") + + userlist = {} + i = 0 + for a in accounts: + # Do not let people try to buffer overflow some busted passwd parser. + if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue + + userlist[a['uid']] = a['gidNumber'] + line = "%s:%s:%d:%d:%s:%s%s:%s" % ( + a['uid'], + PwdMarker, + a['uidNumber'], + a['gidNumber'], + a['gecos'], + HomePrefix, a['uid'], + a['loginShell']) + line = Sanitize(line) + "\n" + F.write("0%u %s" % (i, line)) + F.write(".%s %s" % (a['uid'], line)) + F.write("=%d %s" % (a['uidNumber'], line)) + i = i + 1 + + # Oops, something unspeakable happened. + except: + Die(File, None, F) + raise + Done(File, None, F) + + # Return the list of users so we know which keys to export + return userlist + +def GenAllUsers(accounts, file): + f = None + try: + OldMask = os.umask(0022) + f = open(file + ".tmp", "w", 0644) + os.umask(OldMask) + + all = [] + for a in accounts: + all.append( { 'uid': a['uid'], + 'uidNumber': a['uidNumber'], + 'active': a.pw_active() and a.shadow_active() } ) + json.dump(all, f) + + # Oops, something unspeakable happened. + except: + Die(file, f, None) + raise + Done(file, f, None) # Generate the shadow list -def GenShadow(l,File): - F = None; - try: - OldMask = os.umask(0077); - F = open(File + ".tdb.tmp","w",0600); - os.umask(OldMask); - - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - I = 0; - for x in PasswdAttrs: - if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0: - continue; - - Pass = GetAttr(x,"userPassword"); - if Pass[0:7] != "{crypt}" or len(Pass) > 50: - Pass = '*'; - else: - Pass = Pass[7:]; +def GenShadow(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File + ".tdb.tmp", "w", 0600) + os.umask(OldMask) + + i = 0 + for a in accounts: + # If the account is locked, mark it as such in shadow + # See Debian Bug #308229 for why we set it to 1 instead of 0 + if not a.pw_active(): ShadowExpire = '1' + elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire']) + else: ShadowExpire = '' + + values = [] + values.append(a['uid']) + values.append(a.get_password()) + for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive': + if key in a: values.append(a[key]) + else: values.append('') + values.append(ShadowExpire) + line = ':'.join(values)+':' + line = Sanitize(line) + "\n" + F.write("0%u %s" % (i, line)) + F.write(".%s %s" % (a['uid'], line)) + i = i + 1 + + # Oops, something unspeakable happened. + except: + Die(File, None, F) + raise + Done(File, None, F) - # If the account is locked, mark it as such in shadow - # See Debian Bug #308229 for why we set it to 1 instead of 0 - if (GetAttr(x,"userPassword").find("*LK*") != -1) \ - or GetAttr(x,"userPassword").startswith("!"): - ShadowExpire = '1' - else: - ShadowExpire = GetAttr(x,"shadowExpire") - - Line = "%s:%s:%s:%s:%s:%s:%s:%s:" % (GetAttr(x,"uid"),\ - Pass,GetAttr(x,"shadowLastChange"),\ - GetAttr(x,"shadowMin"),GetAttr(x,"shadowMax"),\ - GetAttr(x,"shadowWarning"),GetAttr(x,"shadowInactive"),\ - ShadowExpire); - Line = Sanitize(Line) + "\n"; - F.write("0%u %s" % (I,Line)); - F.write(".%s %s" % (GetAttr(x,"uid"),Line)); - I = I + 1; - - # Oops, something unspeakable happened. - except: - Die(File,None,F); - raise; - Done(File,None,F); +# Generate the sudo passwd file +def GenShadowSudo(accounts, File, untrusted, current_host): + F = None + try: + OldMask = os.umask(0077) + F = open(File + ".tmp", "w", 0600) + os.umask(OldMask) + + for a in accounts: + Pass = '*' + if 'sudoPassword' in a: + for entry in a['sudoPassword']: + Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry) + if Match == None: + continue + uuid = Match.group(1) + status = Match.group(2) + hosts = Match.group(3) + cryptedpass = Match.group(4) + + if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass): + continue + for_all = hosts == "*" + for_this_host = current_host in hosts.split(',') + if not (for_all or for_this_host): + continue + # ignore * passwords for untrusted hosts, but copy host specific passwords + if for_all and untrusted: + continue + Pass = cryptedpass + if for_this_host: # this makes sure we take a per-host entry over the for-all entry + break + if len(Pass) > 50: + Pass = '*' + + Line = "%s:%s" % (a['uid'], Pass) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) # Generate the sudo passwd file -def GenShadowSudo(l,File): - F = None; - try: - OldMask = os.umask(0077); - F = open(File + ".tmp","w",0600); - os.umask(OldMask); +def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None): + F = None + if sshcommand is None: + sshcommand = GitoliteSSHCommand + try: + OldMask = os.umask(0022) + F = open(File + ".tmp", "w", 0600) + os.umask(OldMask) + + if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "": + for a in accounts: + if not 'sshRSAAuthKey' in a: continue + + User = a['uid'] + prefix = GitoliteSSHRestrictions + prefix = prefix.replace('@@COMMAND@@', sshcommand) + prefix = prefix.replace('@@USER@@', User) + for I in a["sshRSAAuthKey"]: + if I.startswith("allowed_hosts=") and ' ' in line: + if current_host is None: + continue + machines, I = I.split('=', 1)[1].split(' ', 1) + if current_host not in machines.split(','): + continue # skip this key + + if I.startswith('ssh-'): + line = "%s %s"%(prefix, I) + else: + continue # do not allow keys with other restrictions that might conflict + line = Sanitize(line) + "\n" + F.write(line) + + for dn, attrs in hosts: + if not 'sshRSAHostKey' in attrs: continue + hostname = "host-" + attrs['hostname'][0] + prefix = GitoliteSSHRestrictions + prefix = prefix.replace('@@COMMAND@@', sshcommand) + prefix = prefix.replace('@@USER@@', hostname) + for I in attrs["sshRSAHostKey"]: + line = "%s %s"%(prefix, I) + line = Sanitize(line) + "\n" + F.write(line) + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) +# Generate the shadow list +def GenSSHShadow(global_dir, accounts): # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - for x in PasswdAttrs: - if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0: - continue; - if not x[1].has_key('sudoPassword'): + userkeys = {} + + for a in accounts: + if not 'sshRSAAuthKey' in a: continue + + contents = [] + for I in a['sshRSAAuthKey']: + MultipleLine = "%s" % I + MultipleLine = Sanitize(MultipleLine) + contents.append(MultipleLine) + userkeys[a['uid']] = contents + return userkeys + +# Generate the webPassword list +def GenWebPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + for a in accounts: + if not 'webPassword' in a: continue + if not a.pw_active(): continue + + Pass = str(a['webPassword']) + Line = "%s:%s" % (a['uid'], Pass) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + + except: + Die(File, None, F) + raise + +# Generate the rtcPassword list +def GenRtcPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + for a in accounts: + if a.is_guest_account(): continue + if not 'rtcPassword' in a: continue + if not a.pw_active(): continue + + Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + + except: + Die(File, None, F) + raise + +# Generate the TOTP auth file +def GenTOTPSeed(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + F.write("# Option User Prefix Seed\n") + for a in accounts: + if a.is_guest_account(): continue + if not 'totpSeed' in a: continue + if not a.pw_active(): continue + + Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed']) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + except: + Die(File, None, F) + raise + + +def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host): + OldMask = os.umask(0077) + tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz') + os.umask(OldMask) + for f in userlist: + if f not in ssh_userkeys: continue + # If we're not exporting their primary group, don't export + # the key and warn + grname = None + if userlist[f] in grouprevmap.keys(): + grname = grouprevmap[userlist[f]] + else: + try: + if int(userlist[f]) <= 100: + # In these cases, look it up in the normal way so we + # deal with cases where, for instance, users are in group + # users as their primary group. + grname = grp.getgrgid(userlist[f])[0] + except Exception, e: + pass - Pass = None - for entry in x[1]['sudoPassword']: - Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry.lower()) - if Match == None: - continue - uuid = Match.group(1) - status = Match.group(2) - hosts = Match.group(3) - cryptedpass = Match.group(4) + if grname is None: + print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f]) + continue - if status != 'confirmed:'+make_sudopasswd_hmac('password-is-confirmed', uuid, hosts, cryptedpass): - continue - for_all = hosts == "*" - for_this_host = CurrentHost in hosts.split(',') - if not (for_all or for_this_host): - continue - Pass = cryptedpass - if for_this_host: # this makes sure we take a per-host entry over the for-all entry - break - if not Pass: + lines = [] + for line in ssh_userkeys[f]: + if line.startswith("allowed_hosts=") and ' ' in line: + machines, line = line.split('=', 1)[1].split(' ', 1) + if current_host not in machines.split(','): + continue # skip this key + lines.append(line) + if not lines: + continue # no keys for this host + contents = "\n".join(lines) + "\n" + + to = tarfile.TarInfo(name=f) + # These will only be used where the username doesn't + # exist on the target system for some reason; hence, + # in those cases, the safest thing is for the file to + # be owned by root but group nobody. This deals with + # the bloody obscure case where the group fails to exist + # whilst the user does (in which case we want to avoid + # ending up with a file which is owned user:root to avoid + # a fairly obvious attack vector) + to.uid = 0 + to.gid = 65534 + # Using the username / groupname fields avoids any need + # to give a shit^W^W^Wcare about the UIDoffset stuff. + to.uname = f + to.gname = grname + to.mode = 0400 + to.mtime = int(time.time()) + to.size = len(contents) + + tf.addfile(to, StringIO(contents)) + + tf.close() + os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target) + +# add a list of groups to existing groups, +# including all subgroups thereof, recursively. +# basically this proceduces the transitive hull of the groups in +# addgroups. +def addGroups(existingGroups, newGroups, uid, current_host): + for group in newGroups: + # if it's a @host, split it and verify it's on the current host. + s = group.split('@', 1) + if len(s) == 2 and s[1] != current_host: + continue + group = s[0] + + # let's see if we handled this group already + if group in existingGroups: continue - if len(Pass) > 50: + + if not GroupIDMap.has_key(group): + print "Group", group, "does not exist but", uid, "is in it" continue - Line = "%s:%s" % (GetAttr(x,"uid"), Pass) - Line = Sanitize(Line) + "\n"; - F.write("%s" % (Line)); + existingGroups.append(group) - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); + if SubGroupMap.has_key(group): + addGroups(existingGroups, SubGroupMap[group], uid, current_host) -# Generate the shadow list -def GenSSHShadow(l,masterFileName): - # Fetch all the users - singlefile = None - userfiles = [] - # Depending on config, we write out either a single file, - # multiple files, or both - if SingleSSHFile: - try: - OldMask = os.umask(0077); - masterFile = open(masterFileName + ".tmp","w",0600); - os.umask(OldMask); - except IOError: - Die(masterFileName,masterFile,None) - raise - - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - # If we're going to be dealing with multiple keys, empty the - # directory before we start to avoid old keys hanging around - if MultipleSSHFiles: - safe_rmtree(os.path.join(GlobalDir, 'userkeys')) - safe_makedirs(os.path.join(GlobalDir, 'userkeys')) - - for x in PasswdAttrs: - # If the account is locked, do not write it. - # This is a partial stop-gap. The ssh also needs to change this - # to ignore ~/.ssh/authorized* files. - if (GetAttr(x,"userPassword").find("*LK*") != -1) \ - or GetAttr(x,"userPassword").startswith("!"): - continue; - - if x[1].has_key("uidNumber") == 0 or \ - x[1].has_key("sshRSAAuthKey") == 0: - continue; - User = GetAttr(x,"uid"); - F = None; +# Generate the group list +def GenGroup(accounts, File, current_host): + grouprevmap = {} + F = None + try: + F = open(File + ".tdb.tmp", "w") + + # Generate the GroupMap + GroupMap = {} + for x in GroupIDMap: + GroupMap[x] = [] + GroupHasPrimaryMembers = {} + + # Sort them into a list of groups having a set of users + for a in accounts: + GroupHasPrimaryMembers[ a['gidNumber'] ] = True + if not 'supplementaryGid' in a: continue + + supgroups=[] + addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host) + for g in supgroups: + GroupMap[g].append(a['uid']) + + # Output the group file. + J = 0 + for x in GroupMap.keys(): + if not x in GroupIDMap: + continue - try: - if MultipleSSHFiles: - OldMask = os.umask(0077); - File = os.path.join(GlobalDir, 'userkeys', User) - F = open(File + ".tmp","w",0600); - os.umask(OldMask); - - for I in x[1]["sshRSAAuthKey"]: - if MultipleSSHFiles: - MultipleLine = "%s" % I - MultipleLine = Sanitize(MultipleLine) + "\n" - F.write(MultipleLine) - if SingleSSHFile: - SingleLine = "%s: %s" % (User, I) - SingleLine = Sanitize(SingleLine) + "\n" - masterFile.write(SingleLine) - - if MultipleSSHFiles: - Done(File,F,None); - userfiles.append(os.path.basename(File)) - - # Oops, something unspeakable happened. - except IOError: - Die(File,F,None) - Die(masterFileName,masterFile,None) - raise; + if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers: + continue - if SingleSSHFile: - Done(masterFileName,masterFile,None) - singlefile = os.path.basename(masterFileName) + grouprevmap[GroupIDMap[x]] = x - return singlefile, userfiles + Line = "%s:x:%u:" % (x, GroupIDMap[x]) + Comma = '' + for I in GroupMap[x]: + Line = Line + ("%s%s" % (Comma, I)) + Comma = ',' + Line = Sanitize(Line) + "\n" + F.write("0%u %s" % (J, Line)) + F.write(".%s %s" % (x, Line)) + F.write("=%u %s" % (GroupIDMap[x], Line)) + J = J + 1 + + # Oops, something unspeakable happened. + except: + Die(File, None, F) + raise + Done(File, None, F) + + return grouprevmap -# Generate the group list -def GenGroup(l,File): - grouprevmap = {} - F = None; - try: - F = open(File + ".tdb.tmp","w"); - - # Generate the GroupMap - GroupMap = {}; - for x in GroupIDMap.keys(): - GroupMap[x] = []; - - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - # Sort them into a list of groups having a set of users - for x in PasswdAttrs: - if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0: - continue; - if x[1].has_key("supplementaryGid") == 0: - continue; - - for I in x[1]["supplementaryGid"]: - if GroupMap.has_key(I): - GroupMap[I].append(GetAttr(x,"uid")); - else: - print "Group does not exist ",I,"but",GetAttr(x,"uid"),"is in it"; - - # Output the group file. - J = 0; - for x in GroupMap.keys(): - grouprevmap[GroupIDMap[x]] = x - if GroupIDMap.has_key(x) == 0: - continue; - Line = "%s:x:%u:" % (x,GroupIDMap[x]); - Comma = ''; - for I in GroupMap[x]: - Line = Line + ("%s%s" % (Comma,I)); - Comma = ','; - Line = Sanitize(Line) + "\n"; - F.write("0%u %s" % (J,Line)); - F.write(".%s %s" % (x,Line)); - F.write("=%u %s" % (GroupIDMap[x],Line)); - J = J + 1; - - # Oops, something unspeakable happened. - except: - Die(File,None,F); - raise; - Done(File,None,F); - - return grouprevmap +def CheckForward(accounts): + for a in accounts: + if not 'emailForward' in a: continue -# Generate the email forwarding list -def GenForward(l,File): - F = None; - try: - OldMask = os.umask(0022); - F = open(File + ".tmp","w",0644); - os.umask(OldMask); + delete = False - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - # Write out the email address for each user - for x in PasswdAttrs: - if x[1].has_key("emailForward") == 0 or IsInGroup(x) == 0: - continue; - # Do not allow people to try to buffer overflow busted parsers - if len(GetAttr(x,"emailForward")) > 200: - continue; - + if len(a['emailForward']) > 200: delete = True # Check the forwarding address - if EmailCheck.match(GetAttr(x,"emailForward")) == None: - continue; - Line = "%s: %s" % (GetAttr(x,"uid"),GetAttr(x,"emailForward")); - Line = Sanitize(Line) + "\n"; - F.write(Line); - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); - -def GenAllForward(l,File): - Fdb = None; - try: - OldMask = os.umask(0022); - Fdb = os.popen("cdbmake %s %s.tmp"%(File,File),"w"); - os.umask(OldMask); + elif EmailCheck.match(a['emailForward']) is None: delete = True - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - # Write out the email address for each user - for x in PasswdAttrs: - if x[1].has_key("emailForward") == 0: - continue; - - # Do not allow people to try to buffer overflow busted parsers - Forward = GetAttr(x,"emailForward"); - if len(Forward) > 200: - continue; + if delete: + a.delete_mailforward() - # Check the forwarding address - if EmailCheck.match(Forward) == None: - continue; - - User = GetAttr(x,"uid"); - Fdb.write("+%d,%d:%s->%s\n"%(len(User),len(Forward),User,Forward)); - Fdb.write("\n"); - # Oops, something unspeakable happened. - except: - Fdb.close(); - raise; - if Fdb.close() != None: - raise "cdbmake gave an error"; - -# Generate the anon XEarth marker file -def GenMarkers(l,File): - F = None; - try: - F = open(File + ".tmp","w"); +# Generate the email forwarding list +def GenForward(accounts, File): + F = None + try: + OldMask = os.umask(0022) + F = open(File + ".tmp", "w", 0644) + os.umask(OldMask) + + for a in accounts: + if not 'emailForward' in a: continue + Line = "%s: %s" % (a['uid'], a['emailForward']) + Line = Sanitize(Line) + "\n" + F.write(Line) - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - # Write out the position for each user - for x in PasswdAttrs: - if x[1].has_key("latitude") == 0 or x[1].has_key("longitude") == 0: - continue; - try: - Line = "%8s %8s \"\""%(DecDegree(GetAttr(x,"latitude"),1),DecDegree(GetAttr(x,"longitude"),1)); - Line = Sanitize(Line) + "\n"; - F.write(Line); - except: - pass; - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + +def GenCDB(accounts, File, key): + Fdb = None + try: + OldMask = os.umask(0022) + # nothing else does the fsync stuff, so why do it here? + prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else '' + Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w") + os.umask(OldMask) + + # Write out the email address for each user + for a in accounts: + if not key in a: continue + value = a[key] + user = a['uid'] + Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value)) + + Fdb.write("\n") + # Oops, something unspeakable happened. + except: + Fdb.close() + raise + if Fdb.close() != None: + raise "cdbmake gave an error" + +def GenDBM(accounts, File, key): + Fdb = None + OldMask = os.umask(0022) + fn = os.path.join(File).encode('ascii', 'ignore') + try: + posix.remove(fn) + except: + pass + + try: + Fdb = dbm.open(fn, "c") + os.umask(OldMask) + + # Write out the email address for each user + for a in accounts: + if not key in a: continue + value = a[key] + user = a['uid'] + Fdb[user] = value + + Fdb.close() + except: + # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db + os.remove(File + ".db") + raise + # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db + os.rename (File + ".db", File) + +# Generate the anon XEarth marker file +def GenMarkers(accounts, File): + F = None + try: + F = open(File + ".tmp", "w") + + # Write out the position for each user + for a in accounts: + if not ('latitude' in a and 'longitude' in a): continue + try: + Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True)) + Line = Sanitize(Line) + "\n" + F.write(Line) + except: + pass + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) # Generate the debian-private subscription list -def GenPrivate(l,File): - F = None; - try: - F = open(File + ".tmp","w"); +def GenPrivate(accounts, File): + F = None + try: + F = open(File + ".tmp", "w") + + # Write out the position for each user + for a in accounts: + if not a.is_active_user(): continue + if a.is_guest_account(): continue + if not 'privateSub' in a: continue + try: + Line = "%s"%(a['privateSub']) + Line = Sanitize(Line) + "\n" + F.write(Line) + except: + pass + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; +# Generate a list of locked accounts +def GenDisabledAccounts(accounts, File): + F = None + try: + F = open(File + ".tmp", "w") + disabled_accounts = [] + + # Fetch all the users + for a in accounts: + if a.pw_active(): continue + Line = "%s:%s" % (a['uid'], "Account is locked") + disabled_accounts.append(a) + F.write(Sanitize(Line) + "\n") + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + return disabled_accounts - # Write out the position for each user - for x in PasswdAttrs: - if x[1].has_key("privateSub") == 0: - continue; +# Generate the list of local addresses that refuse all mail +def GenMailDisable(accounts, File): + F = None + try: + F = open(File + ".tmp", "w") + + for a in accounts: + if not 'mailDisableMessage' in a: continue + Line = "%s: %s"%(a['uid'], a['mailDisableMessage']) + Line = Sanitize(Line) + "\n" + F.write(Line) - # If the account is locked, do not write it - if (GetAttr(x,"userPassword").find("*LK*") != -1) \ - or GetAttr(x,"userPassword").startswith("!"): - continue; + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) - # If the account has no PGP key, do not write it - if x[1].has_key("keyFingerPrint") == 0: - continue; +# Generate a list of uids that should have boolean affects applied +def GenMailBool(accounts, File, key): + F = None + try: + F = open(File + ".tmp", "w") + + for a in accounts: + if not key in a: continue + if not a[key] == 'TRUE': continue + Line = "%s"%(a['uid']) + Line = Sanitize(Line) + "\n" + F.write(Line) - # Must be in the Debian group (yuk, hard coded for now) - if GetAttr(x,"gidNumber") != "800": - continue; + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) - try: - Line = "%s"%(GetAttr(x,"privateSub")); - Line = Sanitize(Line) + "\n"; - F.write(Line); - except: - pass; - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); +# Generate a list of hosts for RBL or whitelist purposes. +def GenMailList(accounts, File, key): + F = None + try: + F = open(File + ".tmp", "w") -# Generate a list of locked accounts -def GenDisabledAccounts(l,File): - F = None; - try: - F = open(File + ".tmp","w"); + if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$') + else: validregex = re.compile('^[-\w.]+$') - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - I = 0; - for x in PasswdAttrs: - if x[1].has_key("uidNumber") == 0: - continue; - - Pass = GetAttr(x,"userPassword"); - Line = "" - # *LK* is the reference value for a locked account - # password starting with ! is also a locked account - if Pass.find("*LK*") != -1 or Pass.startswith("!"): - # Format is : - Line = "%s:%s" % (GetAttr(x,"uid"), "Account is locked") - - if Line != "": - F.write(Sanitize(Line) + "\n") - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); + for a in accounts: + if not key in a: continue -# Generate the list of local addresses that refuse all mail -def GenMailDisable(l,File): - F = None; - try: - F = open(File + ".tmp","w"); + filtered = filter(lambda z: validregex.match(z), a[key]) + if len(filtered) == 0: continue + if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered) + line = a['uid'] + ': ' + ' : '.join(filtered) + line = Sanitize(line) + "\n" + F.write(line) - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) - for x in PasswdAttrs: - Reason = None +def isRoleAccount(account): + return 'debianRoleAccount' in account['objectClass'] - if x[1].has_key("mailDisableMessage"): - Reason = GetAttr(x,"mailDisableMessage") - else: - continue +# Generate the DNS Zone file +def GenDNS(accounts, File): + F = None + try: + F = open(File + ".tmp", "w") + + # Fetch all the users + RRs = {} + + # Write out the zone file entry for each user + for a in accounts: + if not 'dnsZoneEntry' in a: continue + if not a.is_active_user() and not isRoleAccount(a): continue + if a.is_guest_account(): continue + + try: + F.write("; %s\n"%(a.email_address())) + for z in a["dnsZoneEntry"]: + Split = z.lower().split() + if Split[1].lower() == 'in': + Line = " ".join(Split) + "\n" + F.write(Line) + + Host = Split[0] + DNSZone + if BSMTPCheck.match(Line) != None: + F.write("; Has BSMTP\n") + + # Write some identification information + if not RRs.has_key(Host): + if Split[2].lower() in ["a", "aaaa"]: + Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address()) + for y in a["keyFingerPrint"]: + Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y)) + F.write(Line) + RRs[Host] = 1 + else: + Line = "; Err %s"%(str(Split)) + F.write(Line) + + F.write("\n") + except Exception, e: + F.write("; Errors:\n") + for line in str(e).split("\n"): + F.write("; %s\n"%(line)) + pass + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + +def is_ipv6_addr(i): + try: + socket.inet_pton(socket.AF_INET6, i) + except socket.error: + return False + return True + +def ExtractDNSInfo(x): + hostname = GetAttr(x, "hostname") + + TTLprefix="\t" + if 'dnsTTL' in x[1]: + TTLprefix="%s\t"%(x[1]["dnsTTL"][0]) + + DNSInfo = [] + if x[1].has_key("ipHostNumber"): + for I in x[1]["ipHostNumber"]: + if is_ipv6_addr(I): + DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I)) + else: + DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I)) - # Must be in the Debian group (yuk, hard coded for now) - if GetAttr(x,"gidNumber") != "800": - continue; + Algorithm = None - try: - Line = "%s: %s"%(GetAttr(x,"uid"),Reason); - Line = Sanitize(Line) + "\n"; - F.write(Line); - except: - pass; - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); + ssh_hostnames = [ hostname ] + if x[1].has_key("sshfpHostname"): + ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ] -# Generate a list of uids that should have boolean affects applied -def GenMailBool(l,File,Key): - F = None; - try: - F = open(File + ".tmp","w"); + if 'sshRSAHostKey' in x[1]: + for I in x[1]["sshRSAHostKey"]: + Split = I.split() + key_prefix = Split[0] + key = base64.decodestring(Split[1]) - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - for x in PasswdAttrs: - Reason = None - - if x[1].has_key(Key) == 0: - continue + # RFC4255 + # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml + if key_prefix == 'ssh-rsa': + Algorithm = 1 + if key_prefix == 'ssh-dss': + Algorithm = 2 + if key_prefix == 'ssh-ed25519': + Algorithm = 4 + if Algorithm == None: + continue + # and more from the registry + sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ] + + fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ] + for h in ssh_hostnames: + for digest_codepoint, fingerprint in fingerprints: + DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint)) + + if 'architecture' in x[1]: + Arch = GetAttr(x, "architecture") + Mach = "" + if x[1].has_key("machine"): + Mach = " " + GetAttr(x, "machine") + DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian")) + + if x[1].has_key("mXRecord"): + for I in x[1]["mXRecord"]: + if I in MX_remap: + for e in MX_remap[I]: + DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e)) + else: + DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I)) - # Must be in the Debian group (yuk, hard coded for now) - if GetAttr(x,"gidNumber") != "800": - continue + return DNSInfo - if GetAttr(x,Key) != "TRUE": - continue +# Generate the DNS records +def GenZoneRecords(host_attrs, File): + F = None + try: + F = open(File + ".tmp", "w") - try: - Line = "%s"%(GetAttr(x,"uid")); - Line = Sanitize(Line) + "\n"; - F.write(Line); - except: - pass; - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); + # Fetch all the hosts + for x in host_attrs: + if x[1].has_key("hostname") == 0: + continue -# Generate a list of hosts for RBL or whitelist purposes. -def GenMailList(l,File,Key): - F = None; - try: - F = open(File + ".tmp","w"); + if IsDebianHost.match(GetAttr(x, "hostname")) is None: + continue - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - for x in PasswdAttrs: - Reason = None - - if x[1].has_key(Key) == 0: - continue + for Line in ExtractDNSInfo(x): + F.write(Line + "\n") - # Must be in the Debian group (yuk, hard coded for now) - if GetAttr(x,"gidNumber") != "800": - continue + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) - try: - found = 0 - Line = None - for z in x[1][Key]: - if Key == "mailWhitelist": - if re.match('^[-\w.]+(/[\d]+)?$',z) == None: - continue - else: - if re.match('^[-\w.]+$',z) == None: - continue - if found == 0: - found = 1 - Line = GetAttr(x,"uid") - else: - Line += " " - Line += ": " + z - if Key == "mailRHSBL": - Line += "/$sender_address_domain" - - if Line != None: - Line = Sanitize(Line) + "\n"; - F.write(Line); - except: - pass; - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); +# Generate the BSMTP file +def GenBSMTP(accounts, File, HomePrefix): + F = None + try: + F = open(File + ".tmp", "w") + + # Write out the zone file entry for each user + for a in accounts: + if not 'dnsZoneEntry' in a: continue + if not a.is_active_user(): continue + + try: + for z in a["dnsZoneEntry"]: + Split = z.lower().split() + if Split[1].lower() == 'in': + for y in range(0, len(Split)): + if Split[y] == "$": + Split[y] = "\n\t" + Line = " ".join(Split) + "\n" + + Host = Split[0] + DNSZone + if BSMTPCheck.match(Line) != None: + F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host, + a['uid'], HomePrefix, a['uid'], Host)) + + except: + F.write("; Errors\n") + pass + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + +def HostToIP(Host, mapped=True): -# Generate the DNS Zone file -def GenDNS(l,File,HomePrefix): - F = None; - try: - F = open(File + ".tmp","w"); - + IPAdresses = [] + + if Host[1].has_key("ipHostNumber"): + for addr in Host[1]["ipHostNumber"]: + IPAdresses.append(addr) + if not is_ipv6_addr(addr) and mapped == "True": + IPAdresses.append("::ffff:"+addr) + + return IPAdresses + +# Generate the ssh known hosts file +def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None): + F = None + try: + OldMask = os.umask(0022) + F = open(File + ".tmp", "w", 0644) + os.umask(OldMask) + + for x in host_attrs: + if x[1].has_key("hostname") == 0 or \ + x[1].has_key("sshRSAHostKey") == 0: + continue + Host = GetAttr(x, "hostname") + HostNames = [ Host ] + if Host.endswith(HostDomain): + HostNames.append(Host[:-(len(HostDomain) + 1)]) + + # in the purpose field [[host|some other text]] (where some other text is optional) + # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts + # file. But so that we don't have to add everything we link we can add an asterisk + # and say [[*... to ignore it. In order to be able to add stuff to ssh without + # http linking it we also support [[-hostname]] entries. + for i in x[1].get("purpose", []): + m = PurposeHostField.match(i) + if m: + m = m.group(1) + # we ignore [[*..]] entries + if m.startswith('*'): + continue + if m.startswith('-'): + m = m[1:] + if m: + HostNames.append(m) + if m.endswith(HostDomain): + HostNames.append(m[:-(len(HostDomain) + 1)]) + + for I in x[1]["sshRSAHostKey"]: + if mode and mode == 'authorized_keys': + hosts = HostToIP(x) + if 'sshdistAuthKeysHost' in x[1]: + hosts += x[1]['sshdistAuthKeysHost'] + clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host) + clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand) + Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I) + else: + Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I) + Line = Sanitize(Line) + "\n" + F.write(Line) + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + +# Generate the debianhosts file (list of all IP addresses) +def GenHosts(host_attrs, File): + F = None + try: + OldMask = os.umask(0022) + F = open(File + ".tmp", "w", 0644) + os.umask(OldMask) + + seen = set() + + for x in host_attrs: + + if IsDebianHost.match(GetAttr(x, "hostname")) is None: + continue + + if not 'ipHostNumber' in x[1]: + continue + + addrs = x[1]["ipHostNumber"] + for addr in addrs: + if addr not in seen: + seen.add(addr) + addr = Sanitize(addr) + "\n" + F.write(addr) + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + +def replaceTree(src, dst_basedir): + bn = os.path.basename(src) + dst = os.path.join(dst_basedir, bn) + safe_rmtree(dst) + shutil.copytree(src, dst) + +def GenKeyrings(OutDir): + for k in Keyrings: + if os.path.isdir(k): + replaceTree(k, OutDir) + else: + shutil.copy(k, OutDir) + + +def get_accounts(ldap_conn): # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - # Write out the zone file entry for each user - for x in PasswdAttrs: - if x[1].has_key("dnsZoneEntry") == 0: - continue; - - # If the account has no PGP key, do not write it - if x[1].has_key("keyFingerPrint") == 0: - continue; - try: - F.write("; %s\n"%(EmailAddress(x))); - for z in x[1]["dnsZoneEntry"]: - Split = z.lower().split() - if Split[1].lower() == 'in': - for y in range(0,len(Split)): - if Split[y] == "$": - Split[y] = "\n\t"; - Line = " ".join(Split) + "\n"; - F.write(Line); - - Host = Split[0] + DNSZone; - if BSMTPCheck.match(Line) != None: - F.write("; Has BSMTP\n"); - - # Write some identification information - if Split[2].lower() == "a": - Line = "%s IN TXT \"%s\"\n"%(Split[0],EmailAddress(x)); - for y in x[1]["keyFingerPrint"]: - Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0],FormatPGPKey(y)); - F.write(Line); - else: - Line = "; Err %s"%(str(Split)); - F.write(Line); - - F.write("\n"); - except: - F.write("; Errors\n"); - pass; - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); - -# Generate the DNS SSHFP records -def GenSSHFP(l,File,HomePrefix): - F = None - try: - F = open(File + ".tmp","w") - + passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\ + ["uid", "uidNumber", "gidNumber", "supplementaryGid",\ + "gecos", "loginShell", "userPassword", "shadowLastChange",\ + "shadowMin", "shadowMax", "shadowWarning", "shadowInactive", + "shadowExpire", "emailForward", "latitude", "longitude",\ + "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\ + "keyFingerPrint", "privateSub", "mailDisableMessage",\ + "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ + "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ + "mailContentInspectionAction", "webPassword", "rtcPassword",\ + "bATVToken", "totpSeed"]) + + if passwd_attrs is None: + raise UDEmptyList, "No Users" + accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs) + accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower())) + + return accounts + +def get_hosts(ldap_conn): # Fetch all the hosts - global HostAttrs + HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\ + ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\ + "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture", + "sshfpHostname"]) + if HostAttrs == None: - raise "No Hosts" + raise UDEmptyList, "No Hosts" + + HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower())) + + return HostAttrs + + +def make_ldap_conn(): + # Connect to the ldap server + l = connectLDAP() + # for testing purposes it's sometimes useful to pass username/password + # via the environment + if 'UD_CREDENTIALS' in os.environ: + Pass = os.environ['UD_CREDENTIALS'].split() + else: + F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r") + Pass = F.readline().strip().split(" ") + F.close() + l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1]) + + return l - for x in HostAttrs: - if x[1].has_key("hostname") == 0 or \ - x[1].has_key("sshRSAHostKey") == 0: + + +def setup_group_maps(l): + # Fetch all the groups + group_id_map = {} + subgroup_map = {} + attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\ + ["gid", "gidNumber", "subGroup"]) + + # Generate the subgroup_map and group_id_map + for x in attrs: + if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled": continue - Host = GetAttr(x,"hostname"); - Algorithm = None - for I in x[1]["sshRSAHostKey"]: - Split = I.split() - if Split[0] == 'ssh-rsa': - Algorithm = 1 - if Split[0] == 'ssh-dss': - Algorithm = 2 - if Algorithm == None: - continue - Fingerprint = sha.new(base64.decodestring(Split[1])).hexdigest() - Line = "%s. IN SSHFP %u 1 %s" % (Host,Algorithm,Fingerprint) - Line = Sanitize(Line) + "\n" - F.write(Line) - # Oops, something unspeakable happened. - except: - Die(File,F,None) - raise; - Done(File,F,None) + if x[1].has_key("gidNumber") == 0: + continue + group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]) + if x[1].has_key("subGroup") != 0: + subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"]) + + global SubGroupMap + global GroupIDMap + SubGroupMap = subgroup_map + GroupIDMap = group_id_map + +def generate_all(global_dir, ldap_conn): + accounts = get_accounts(ldap_conn) + host_attrs = get_hosts(ldap_conn) + + global_dir += '/' + # Generate global things + accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts") + + accounts = filter(lambda x: not IsRetired(x), accounts) + + CheckForward(accounts) + + GenMailDisable(accounts, global_dir + "mail-disable") + GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward') + GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward') + GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') + GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction') + GenPrivate(accounts, global_dir + "debian-private") + GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock') + GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting") + GenMailBool(accounts, global_dir + "mail-callout", "mailCallout") + GenMailList(accounts, global_dir + "mail-rbl", "mailRBL") + GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL") + GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist") + GenWebPassword(accounts, global_dir + "web-passwords") + GenRtcPassword(accounts, global_dir + "rtc-passwords") + GenTOTPSeed(accounts, global_dir + "users.oath") + GenKeyrings(global_dir) -# Generate the BSMTP file -def GenBSMTP(l,File,HomePrefix): - F = None; - try: - F = open(File + ".tmp","w"); - - # Fetch all the users - global PasswdAttrs; - if PasswdAttrs == None: - raise "No Users"; - - # Write out the zone file entry for each user - for x in PasswdAttrs: - if x[1].has_key("dnsZoneEntry") == 0: - continue; - - # If the account has no PGP key, do not write it - if x[1].has_key("keyFingerPrint") == 0: - continue; - try: - for z in x[1]["dnsZoneEntry"]: - Split = z.lower().split() - if Split[1].lower() == 'in': - for y in range(0,len(Split)): - if Split[y] == "$": - Split[y] = "\n\t"; - Line = " ".join(Split) + "\n"; - - Host = Split[0] + DNSZone; - if BSMTPCheck.match(Line) != None: - F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host, - GetAttr(x,"uid"),HomePrefix,GetAttr(x,"uid"),Host)); - - except: - F.write("; Errors\n"); - pass; - - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); - -# cache IP adresses -HostToIPCache = {} -def HostToIP(Host): - global HostToIPCache - if not Host in HostToIPCache: - IPAdressesT = None - try: - IPAdressesT = list(set([ (a[0],a[4][0]) for a in socket.getaddrinfo(Host, None)])) - except socket.gaierror, (code): - if code[0] != -2: raise - IPAdresses = [] - for addr in IPAdressesT: - if addr[0] == socket.AF_INET: IPAdresses += [addr[1], "::ffff:"+addr[1]] - else: IPAdresses += [addr[1]] - HostToIPCache[Host] = IPAdresses - return HostToIPCache[Host] + # Compatibility. + GenForward(accounts, global_dir + "forward-alias") + GenAllUsers(accounts, global_dir + 'all-accounts.json') + accounts = filter(lambda a: not a in accounts_disabled, accounts) -# Generate the ssh known hosts file -def GenSSHKnown(l,File,mode=None): - F = None; - try: - OldMask = os.umask(0022); - F = open(File + ".tmp","w",0644); - os.umask(OldMask); - - global HostAttrs - if HostAttrs == None: - raise "No Hosts"; - - for x in HostAttrs: - if x[1].has_key("hostname") == 0 or \ - x[1].has_key("sshRSAHostKey") == 0: - continue; - Host = GetAttr(x,"hostname"); - HostNames = [ Host ] - SHost = Host.find(".") - if SHost != None: HostNames += [Host[0:SHost]] + ssh_userkeys = GenSSHShadow(global_dir, accounts) + GenMarkers(accounts, global_dir + "markers") + GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts") + GenHosts(host_attrs, global_dir + "debianhosts") - for I in x[1]["sshRSAHostKey"]: - if mode and mode == 'authorized_keys': - #Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (Host, ",".join(HNames + HostToIP(Host)), I) - Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding %s' % (Host,I) - else: - Line = "%s %s" %(",".join(HostNames + HostToIP(Host)), I); - Line = Sanitize(Line) + "\n"; - F.write(Line); - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); + GenDNS(accounts, global_dir + "dns-zone") + GenZoneRecords(host_attrs, global_dir + "dns-sshfp") -# Generate the debianhosts file (list of all IP addresses) -def GenHosts(l,File): - F = None; - try: - OldMask = os.umask(0022); - F = open(File + ".tmp","w",0644); - os.umask(OldMask); + setup_group_maps(ldap_conn) - # Fetch all the hosts - HostNames = l.search_s(HostBaseDn,ldap.SCOPE_ONELEVEL,"hostname=*",\ - ["hostname"]); - - if HostNames == None: - raise "No Hosts"; - - for x in HostNames: - if x[1].has_key("hostname") == 0: - continue; - Host = GetAttr(x,"hostname"); - try: - Addr = socket.gethostbyname(Host); - F.write(Addr + "\n"); - except: - pass - # Oops, something unspeakable happened. - except: - Die(File,F,None); - raise; - Done(File,F,None); - -def GenKeyrings(l,OutDir): - for k in Keyrings: - shutil.copy(k, OutDir) - -# Connect to the ldap server -l = connectLDAP() -F = open(PassDir+"/pass-"+pwd.getpwuid(os.getuid())[0],"r"); -Pass = F.readline().strip().split(" ") -F.close(); -l.simple_bind_s("uid="+Pass[0]+","+BaseDn,Pass[1]); - -# Fetch all the groups -GroupIDMap = {}; -Attrs = l.search_s(BaseDn,ldap.SCOPE_ONELEVEL,"gid=*",\ - ["gid","gidNumber"]); - -# Generate the GroupMap and GroupIDMap -for x in Attrs: - if x[1].has_key("gidNumber") == 0: - continue; - GroupIDMap[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]); - -# Fetch all the users -PasswdAttrs = l.search_s(BaseDn,ldap.SCOPE_ONELEVEL,"uid=*",\ - ["uid","uidNumber","gidNumber","supplementaryGid",\ - "gecos","loginShell","userPassword","shadowLastChange",\ - "shadowMin","shadowMax","shadowWarning","shadowInactive", - "shadowExpire","emailForward","latitude","longitude",\ - "allowedHost","sshRSAAuthKey","dnsZoneEntry","cn","sn",\ - "keyFingerPrint","privateSub","mailDisableMessage",\ - "mailGreylisting","mailCallout","mailRBL","mailRHSBL",\ - "mailWhitelist", "sudoPassword"]); -# Fetch all the hosts -HostAttrs = l.search_s(HostBaseDn,ldap.SCOPE_ONELEVEL,"sshRSAHostKey=*",\ - ["hostname","sshRSAHostKey"]); - -# Open the control file -if len(sys.argv) == 1: - F = open(GenerateConf,"r"); -else: - F = open(sys.argv[1],"r") - -# Generate global things -GlobalDir = GenerateDir+"/"; -SSHGlobal, SSHFiles = GenSSHShadow(l,GlobalDir+"ssh-rsa-shadow"); -GenAllForward(l,GlobalDir+"mail-forward.cdb"); -GenMarkers(l,GlobalDir+"markers"); -GenPrivate(l,GlobalDir+"debian-private"); -GenDisabledAccounts(l,GlobalDir+"disabled-accounts"); -GenSSHKnown(l,GlobalDir+"ssh_known_hosts"); -#GenSSHKnown(l,GlobalDir+"authorized_keys", 'authorized_keys'); -GenHosts(l,GlobalDir+"debianhosts"); -GenMailDisable(l,GlobalDir+"mail-disable"); -GenMailBool(l,GlobalDir+"mail-greylist","mailGreylisting"); -GenMailBool(l,GlobalDir+"mail-callout","mailCallout"); -GenMailList(l,GlobalDir+"mail-rbl","mailRBL"); -GenMailList(l,GlobalDir+"mail-rhsbl","mailRHSBL"); -GenMailList(l,GlobalDir+"mail-whitelist","mailWhitelist"); -GenKeyrings(l,GlobalDir); - -# Compatibility. -GenForward(l,GlobalDir+"forward-alias"); - -while(1): - Line = F.readline(); - if Line == "": - break; - Line = Line.strip() - if Line == "": - continue; - if Line[0] == '#': - continue; - - Split = Line.split(" ") - OutDir = GenerateDir + '/' + Split[0] + '/'; - try: os.mkdir(OutDir); - except: pass; + for host in host_attrs: + if not "hostname" in host[1]: + continue + generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys) + +def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): + current_host = host[1]['hostname'][0] + OutDir = global_dir + current_host + '/' + if not os.path.isdir(OutDir): + os.mkdir(OutDir) # Get the group list and convert any named groups to numerics - GroupList = {}; - ExtraList = {}; - for I in Split[2:]: - if I[0] == '[': - ExtraList[I] = None; - continue; - GroupList[I] = None; - if GroupIDMap.has_key(I): - GroupList[str(GroupIDMap[I])] = None; - - Allowed = GroupList; - if Allowed == {}: - Allowed = None - CurrentHost = Split[0]; - - # If we're using a single SSH file, deal with it - if SSHGlobal is not None: - DoLink(GlobalDir, OutDir, SSHGlobal) - - DoLink(GlobalDir,OutDir,"debianhosts"); - DoLink(GlobalDir,OutDir,"ssh_known_hosts"); - DoLink(GlobalDir,OutDir,"disabled-accounts") - - sys.stdout.flush(); - if ExtraList.has_key("[NOPASSWD]"): - userlist = GenPasswd(l,OutDir+"passwd",Split[1], "*"); + GroupList = {} + for groupname in AllowedGroupsPreload.strip().split(" "): + GroupList[groupname] = True + if 'allowedGroups' in host[1]: + for groupname in host[1]['allowedGroups']: + GroupList[groupname] = True + for groupname in GroupList.keys(): + if groupname in GroupIDMap: + GroupList[str(GroupIDMap[groupname])] = True + + ExtraList = {} + if 'exportOptions' in host[1]: + for extra in host[1]['exportOptions']: + ExtraList[extra.upper()] = True + + if GroupList != {}: + accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts) + + DoLink(global_dir, OutDir, "debianhosts") + DoLink(global_dir, OutDir, "ssh_known_hosts") + DoLink(global_dir, OutDir, "disabled-accounts") + + sys.stdout.flush() + if 'NOPASSWD' in ExtraList: + userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*") else: - userlist = GenPasswd(l,OutDir+"passwd",Split[1], "x"); - sys.stdout.flush(); - grouprevmap = GenGroup(l,OutDir+"group"); - GenShadowSudo(l, OutDir+"sudo-passwd") + userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x") + sys.stdout.flush() + grouprevmap = GenGroup(accounts, OutDir + "group", current_host) + GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host) # Now we know who we're allowing on the machine, export # the relevant ssh keys - if MultipleSSHFiles: - OldMask = os.umask(0077); - tf = tarfile.open(name=os.path.join(GlobalDir, 'ssh-keys-%s.tar.gz' % CurrentHost), mode='w:gz') - os.umask(OldMask); - for f in userlist.keys(): - if f not in SSHFiles: - continue - # If we're not exporting their primary group, don't export - # the key and warn - grname = None - if userlist[f] in grouprevmap.keys(): - grname = grouprevmap[userlist[f]] - else: - try: - if int(userlist[f]) <= 100: - # In these cases, look it up in the normal way so we - # deal with cases where, for instance, users are in group - # users as their primary group. - grname = grp.getgrgid(userlist[f])[0] - except Exception, e: - pass - - if grname is None: - print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, CurrentHost, userlist[f]) - continue - - to = tf.gettarinfo(os.path.join(GlobalDir, 'userkeys', f), f) - # These will only be used where the username doesn't - # exist on the target system for some reason; hence, - # in those cases, the safest thing is for the file to - # be owned by root but group nobody. This deals with - # the bloody obscure case where the group fails to exist - # whilst the user does (in which case we want to avoid - # ending up with a file which is owned user:root to avoid - # a fairly obvious attack vector) - to.uid = 0 - to.gid = 65534 - # Using the username / groupname fields avoids any need - # to give a shit^W^W^Wcare about the UIDoffset stuff. - to.uname = f - to.gname = grname - to.mode = 0400 - tf.addfile(to, file(os.path.join(GlobalDir, 'userkeys', f))) - - tf.close() - os.rename(os.path.join(GlobalDir, 'ssh-keys-%s.tar.gz' % CurrentHost), - os.path.join(OutDir, 'ssh-keys.tar.gz')) - - if ExtraList.has_key("[UNTRUSTED]"): - continue; - if not ExtraList.has_key("[NOPASSWD]"): - GenShadow(l,OutDir+"shadow"); - - # Link in global things - DoLink(GlobalDir,OutDir,"markers"); - DoLink(GlobalDir,OutDir,"mail-forward.cdb"); - DoLink(GlobalDir,OutDir,"mail-disable"); - DoLink(GlobalDir,OutDir,"mail-greylist"); - DoLink(GlobalDir,OutDir,"mail-callout"); - DoLink(GlobalDir,OutDir,"mail-rbl"); - DoLink(GlobalDir,OutDir,"mail-rhsbl"); - DoLink(GlobalDir,OutDir,"mail-whitelist"); + GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host) + + if not 'NOPASSWD' in ExtraList: + GenShadow(accounts, OutDir + "shadow") + + # Link in global things + if not 'NOMARKERS' in ExtraList: + DoLink(global_dir, OutDir, "markers") + DoLink(global_dir, OutDir, "mail-forward.cdb") + DoLink(global_dir, OutDir, "mail-forward.db") + DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb") + DoLink(global_dir, OutDir, "mail-contentinspectionaction.db") + DoLink(global_dir, OutDir, "mail-disable") + DoLink(global_dir, OutDir, "mail-greylist") + DoLink(global_dir, OutDir, "mail-callout") + DoLink(global_dir, OutDir, "mail-rbl") + DoLink(global_dir, OutDir, "mail-rhsbl") + DoLink(global_dir, OutDir, "mail-whitelist") + DoLink(global_dir, OutDir, "all-accounts.json") + GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward') + GenDBM(accounts, OutDir + "user-forward.db", 'emailForward') + GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken') + GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken') + GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions') + GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions') # Compatibility. - DoLink(GlobalDir,OutDir,"forward-alias"); + DoLink(global_dir, OutDir, "forward-alias") + + if 'DNS' in ExtraList: + DoLink(global_dir, OutDir, "dns-zone") + DoLink(global_dir, OutDir, "dns-sshfp") + + if 'AUTHKEYS' in ExtraList: + DoLink(global_dir, OutDir, "authorized_keys") + + if 'BSMTP' in ExtraList: + GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix) + + if 'PRIVATE' in ExtraList: + DoLink(global_dir, OutDir, "debian-private") + + if 'GITOLITE' in ExtraList: + GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host) + if 'exportOptions' in host[1]: + for entry in host[1]['exportOptions']: + v = entry.split('=',1) + if v[0] != 'GITOLITE' or len(v) != 2: continue + options = v[1].split(',') + group = options.pop(0); + gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts) + if not 'nohosts' in options: + gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts) + else: + gitolite_hosts = [] + command = None + for opt in options: + if opt.startswith('sshcmd='): + command = opt.split('=',1)[1] + GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host) + + if 'WEB-PASSWORDS' in ExtraList: + DoLink(global_dir, OutDir, "web-passwords") - if ExtraList.has_key("[DNS]"): - GenDNS(l,OutDir+"dns-zone",Split[1]); - GenSSHFP(l,OutDir+"dns-sshfp",Split[1]) - - if ExtraList.has_key("[BSMTP]"): - GenBSMTP(l,OutDir+"bsmtp",Split[1]) + if 'RTC-PASSWORDS' in ExtraList: + DoLink(global_dir, OutDir, "rtc-passwords") - if ExtraList.has_key("[PRIVATE]"): - DoLink(GlobalDir,OutDir,"debian-private") + if 'TOTP' in ExtraList: + DoLink(global_dir, OutDir, "users.oath") - if ExtraList.has_key("[KEYRING]"): + if 'KEYRING' in ExtraList: for k in Keyrings: - DoLink(GlobalDir,OutDir,os.path.basename(k)) + bn = os.path.basename(k) + if os.path.isdir(k): + src = os.path.join(global_dir, bn) + replaceTree(src, OutDir) + else: + DoLink(global_dir, OutDir, bn) else: - for k in Keyrings: - try: posix.remove(OutDir+os.path.basename(k)); - except: pass; + for k in Keyrings: + try: + bn = os.path.basename(k) + target = os.path.join(OutDir, bn) + if os.path.isdir(target): + safe_rmtree(dst) + else: + posix.remove(target) + except: + pass + DoLink(global_dir, OutDir, "last_update.trace") + + +def getLastLDAPChangeTime(l): + mods = l.search_s('cn=log', + ldap.SCOPE_ONELEVEL, + '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))', + ['reqEnd']) + + last = 0 + + # Sort the list by reqEnd + sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0]) + # Take the last element in the array + last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0] + + return last + +def getLastKeyringChangeTime(): + krmod = 0 + for k in Keyrings: + mt = os.path.getmtime(k) + if mt > krmod: + krmod = mt + + return int(krmod) + +def getLastBuildTime(gdir): + cache_last_ldap_mod = 0 + cache_last_unix_mod = 0 + cache_last_run = 0 + + try: + fd = open(os.path.join(gdir, "last_update.trace"), "r") + cache_last_mod=fd.read().split() + try: + cache_last_ldap_mod = cache_last_mod[0] + cache_last_unix_mod = int(cache_last_mod[1]) + cache_last_run = int(cache_last_mod[2]) + except IndexError, ValueError: + pass + fd.close() + except IOError, e: + if e.errno == errno.ENOENT: + pass + else: + raise e + + return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run) + +def mq_notify(options, message): + options.section = 'dsa-udgenerate' + options.config = '/etc/dsa/pubsub.conf' + + config = Config(options) + conf = { + 'rabbit_userid': config.username, + 'rabbit_password': config.password, + 'rabbit_virtual_host': config.vhost, + 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'], + 'use_ssl': False + } + + msg = { + 'message': message, + 'timestamp': int(time.time()) + } + conn = None + try: + conn = Connection(conf=conf) + conn.topic_send(config.topic, + json.dumps(msg), + exchange_name=config.exchange, + timeout=5) + finally: + if conn: + conn.close() + +def ud_generate(): + parser = optparse.OptionParser() + parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR", + help="Output directory.") + parser.add_option("-f", "--force", dest="force", action="store_true", + help="Force generation, even if no update to LDAP has happened.") + + (options, args) = parser.parse_args() + if len(args) > 0: + parser.print_help() + sys.exit(1) + + if options.generatedir is not None: + generate_dir = os.environ['UD_GENERATEDIR'] + elif 'UD_GENERATEDIR' in os.environ: + generate_dir = os.environ['UD_GENERATEDIR'] + else: + generate_dir = GenerateDir + + + lockf = os.path.join(generate_dir, 'ud-generate.lock') + lock = get_lock( lockf ) + if lock is None: + sys.stderr.write("Could not acquire lock %s.\n"%(lockf)) + sys.exit(1) + + l = make_ldap_conn() + + time_started = int(time.time()) + ldap_last_mod = getLastLDAPChangeTime(l) + unix_last_mod = getLastKeyringChangeTime() + cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir) + + need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE) + + fd = open(os.path.join(generate_dir, "last_update.trace"), "w") + if need_update or options.force: + msg = 'Update forced' if options.force else 'Update needed' + generate_all(generate_dir, l) + if use_mq: + mq_notify(options, msg) + last_run = int(time.time()) + fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run)) + fd.close() + sys.exit(0) + + +if __name__ == "__main__": + if 'UD_PROFILE' in os.environ: + import cProfile + import pstats + cProfile.run('ud_generate()', "udg_prof") + p = pstats.Stats('udg_prof') + ##p.sort_stats('time').print_stats() + p.sort_stats('cumulative').print_stats() + else: + ud_generate() + +# vim:set et: +# vim:set ts=3: +# vim:set shiftwidth=3: