#!/usr/bin/env python # -*- mode: python -*- # Generates passwd, shadow and group files from the ldap directory. # Copyright (c) 2000-2001 Jason Gunthorpe # Copyright (c) 2003-2004 James Troup # Copyright (c) 2004-2005,7 Joey Schulze # Copyright (c) 2001-2007 Ryan Murray # Copyright (c) 2008,2009,2010,2011 Peter Palfrader # Copyright (c) 2008 Andreas Barth # Copyright (c) 2008 Mark Hymers # Copyright (c) 2008 Luk Claes # Copyright (c) 2008 Thomas Viehmann # Copyright (c) 2009 Stephen Gran # Copyright (c) 2010 Helmut Grohne # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. from dsa_mq.connection import Connection from dsa_mq.config import Config import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm from userdir_ldap import * from userdir_exceptions import * import UDLdap from xml.etree.ElementTree import Element, SubElement, Comment from xml.etree import ElementTree from xml.dom import minidom try: from cStringIO import StringIO except ImportError: from StringIO import StringIO try: import simplejson as json except ImportError: import json if not '__author__' in json.__dict__: sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n") sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n") if os.getuid() == 0: sys.stderr.write("You should probably not run ud-generate as root.\n") sys.exit(1) # # GLOBAL STATE # GroupIDMap = None SubGroupMap = None UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' MAX_UD_AGE = 3600*24 EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$") BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL) PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]") IsDebianHost = re.compile(ConfModule.dns_hostmatch) isSSHFP = re.compile("^\s*IN\s+SSHFP") DNSZone = ".debian.net" Keyrings = ConfModule.sync_keyrings.split(":") GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None) GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None) GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", ".")) MX_remap = json.loads(ConfModule.MX_remap) use_mq = getattr(ConfModule, "use_mq", True) rtc_realm = getattr(ConfModule, "rtc_realm", None) rtc_append = getattr(ConfModule, "rtc_append", None) def prettify(elem): """Return a pretty-printed XML string for the Element. """ rough_string = ElementTree.tostring(elem, 'utf-8') reparsed = minidom.parseString(rough_string) return reparsed.toprettyxml(indent=" ") def safe_makedirs(dir): try: os.makedirs(dir) except OSError, e: if e.errno == errno.EEXIST: pass else: raise e def safe_rmtree(dir): try: shutil.rmtree(dir) except OSError, e: if e.errno == errno.ENOENT: pass else: raise e def get_lock(fn, wait=5*60): f = open(fn, "w") sl = 0.1 ends = time.time() + wait while True: success = False try: fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) return f except IOError: pass if time.time() >= ends: return None sl = min(sl*2, 10, ends - time.time()) time.sleep(sl) return None def Sanitize(Str): return Str.translate(string.maketrans("\n\r\t", "$$$")) def DoLink(From, To, File): try: posix.remove(To + File) except: pass posix.link(From + File, To + File) def IsRetired(account): """ Looks for accountStatus in the LDAP record and tries to match it against one of the known retired statuses """ status = account['accountStatus'] line = status.split() status = line[0] if status == "inactive": return True elif status == "memorial": return True elif status == "retiring": # We'll give them a few extra days over what we said age = 6 * 31 * 24 * 60 * 60 try: return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age except IndexError: return False except ValueError: return False return False # See if this user is in the group list def IsInGroup(account, allowed, current_host): # See if the primary group is in the list if str(account['gidNumber']) in allowed: return True # Check the host based ACL if account.is_allowed_by_hostacl(current_host): return True # See if there are supplementary groups if not 'supplementaryGid' in account: return False supgroups=[] addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host) for g in supgroups: if g in allowed: return True return False def Die(File, F, Fdb): if F != None: F.close() if Fdb != None: Fdb.close() try: os.remove(File + ".tmp") except: pass try: os.remove(File + ".tdb.tmp") except: pass def Done(File, F, Fdb): if F != None: F.close() os.rename(File + ".tmp", File) if Fdb != None: Fdb.close() os.rename(File + ".tdb.tmp", File + ".tdb") # Generate the password list def GenPasswd(accounts, File, HomePrefix, PwdMarker): F = None try: F = open(File + ".tdb.tmp", "w") userlist = {} i = 0 for a in accounts: # Do not let people try to buffer overflow some busted passwd parser. if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue userlist[a['uid']] = a['gidNumber'] line = "%s:%s:%d:%d:%s:%s%s:%s" % ( a['uid'], PwdMarker, a['uidNumber'], a['gidNumber'], a['gecos'], HomePrefix, a['uid'], a['loginShell']) line = Sanitize(line) + "\n" F.write("0%u %s" % (i, line)) F.write(".%s %s" % (a['uid'], line)) F.write("=%d %s" % (a['uidNumber'], line)) i = i + 1 # Oops, something unspeakable happened. except: Die(File, None, F) raise Done(File, None, F) # Return the list of users so we know which keys to export return userlist def GenAllUsers(accounts, file): f = None try: OldMask = os.umask(0022) f = open(file + ".tmp", "w", 0644) os.umask(OldMask) all = [] for a in accounts: all.append( { 'uid': a['uid'], 'uidNumber': a['uidNumber'], 'active': a.pw_active() and a.shadow_active() } ) json.dump(all, f) # Oops, something unspeakable happened. except: Die(file, f, None) raise Done(file, f, None) # Generate the shadow list def GenShadow(accounts, File): F = None try: OldMask = os.umask(0077) F = open(File + ".tdb.tmp", "w", 0600) os.umask(OldMask) i = 0 for a in accounts: # If the account is locked, mark it as such in shadow # See Debian Bug #308229 for why we set it to 1 instead of 0 if not a.pw_active(): ShadowExpire = '1' elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire']) else: ShadowExpire = '' values = [] values.append(a['uid']) values.append(a.get_password()) for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive': if key in a: values.append(a[key]) else: values.append('') values.append(ShadowExpire) line = ':'.join(values)+':' line = Sanitize(line) + "\n" F.write("0%u %s" % (i, line)) F.write(".%s %s" % (a['uid'], line)) i = i + 1 # Oops, something unspeakable happened. except: Die(File, None, F) raise Done(File, None, F) # Generate the sudo passwd file def GenShadowSudo(accounts, File, untrusted, current_host): F = None try: OldMask = os.umask(0077) F = open(File + ".tmp", "w", 0600) os.umask(OldMask) for a in accounts: Pass = '*' if 'sudoPassword' in a: for entry in a['sudoPassword']: Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry) if Match == None: continue uuid = Match.group(1) status = Match.group(2) hosts = Match.group(3) cryptedpass = Match.group(4) if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass): continue for_all = hosts == "*" for_this_host = current_host in hosts.split(',') if not (for_all or for_this_host): continue # ignore * passwords for untrusted hosts, but copy host specific passwords if for_all and untrusted: continue Pass = cryptedpass if for_this_host: # this makes sure we take a per-host entry over the for-all entry break if len(Pass) > 50: Pass = '*' Line = "%s:%s" % (a['uid'], Pass) Line = Sanitize(Line) + "\n" F.write("%s" % (Line)) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate the sudo passwd file def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None): F = None if sshcommand is None: sshcommand = GitoliteSSHCommand try: OldMask = os.umask(0022) F = open(File + ".tmp", "w", 0600) os.umask(OldMask) if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "": for a in accounts: if not 'sshRSAAuthKey' in a: continue User = a['uid'] prefix = GitoliteSSHRestrictions prefix = prefix.replace('@@COMMAND@@', sshcommand) prefix = prefix.replace('@@USER@@', User) for I in a["sshRSAAuthKey"]: if I.startswith("allowed_hosts=") and ' ' in line: if current_host is None: continue machines, I = I.split('=', 1)[1].split(' ', 1) if current_host not in machines.split(','): continue # skip this key if I.startswith('ssh-'): line = "%s %s"%(prefix, I) else: continue # do not allow keys with other restrictions that might conflict line = Sanitize(line) + "\n" F.write(line) for dn, attrs in hosts: if not 'sshRSAHostKey' in attrs: continue hostname = "host-" + attrs['hostname'][0] prefix = GitoliteSSHRestrictions prefix = prefix.replace('@@COMMAND@@', sshcommand) prefix = prefix.replace('@@USER@@', hostname) for I in attrs["sshRSAHostKey"]: line = "%s %s"%(prefix, I) line = Sanitize(line) + "\n" F.write(line) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate the shadow list def GenSSHShadow(global_dir, accounts): # Fetch all the users userkeys = {} for a in accounts: if not 'sshRSAAuthKey' in a: continue contents = [] for I in a['sshRSAAuthKey']: MultipleLine = "%s" % I MultipleLine = Sanitize(MultipleLine) contents.append(MultipleLine) userkeys[a['uid']] = contents return userkeys # Generate the webPassword list def GenWebPassword(accounts, File): F = None try: OldMask = os.umask(0077) F = open(File, "w", 0600) os.umask(OldMask) for a in accounts: if not 'webPassword' in a: continue if not a.pw_active(): continue Pass = str(a['webPassword']) Line = "%s:%s" % (a['uid'], Pass) Line = Sanitize(Line) + "\n" F.write("%s" % (Line)) except: Die(File, None, F) raise # Generate the rtcPassword list def GenRtcPassword(accounts, File): F = None try: OldMask = os.umask(0077) F = open(File, "w", 0600) os.umask(OldMask) for a in accounts: if a.is_guest_account(): continue if not 'rtcPassword' in a: continue if not a.pw_active(): continue Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm) Line = Sanitize(Line) + "\n" F.write("%s" % (Line)) except: Die(File, None, F) raise # Generate the TOTP auth file def GenTOTPSeed(accounts, File): F = None try: OldMask = os.umask(0077) F = open(File, "w", 0600) os.umask(OldMask) F.write("# Option User Prefix Seed\n") for a in accounts: if a.is_guest_account(): continue if not 'totpSeed' in a: continue if not a.pw_active(): continue Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed']) Line = Sanitize(Line) + "\n" F.write("%s" % (Line)) except: Die(File, None, F) raise def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host): OldMask = os.umask(0077) tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz') os.umask(OldMask) for f in userlist: if f not in ssh_userkeys: continue # If we're not exporting their primary group, don't export # the key and warn grname = None if userlist[f] in grouprevmap.keys(): grname = grouprevmap[userlist[f]] else: try: if int(userlist[f]) <= 100: # In these cases, look it up in the normal way so we # deal with cases where, for instance, users are in group # users as their primary group. grname = grp.getgrgid(userlist[f])[0] except Exception, e: pass if grname is None: print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f]) continue lines = [] for line in ssh_userkeys[f]: if line.startswith("allowed_hosts=") and ' ' in line: machines, line = line.split('=', 1)[1].split(' ', 1) if current_host not in machines.split(','): continue # skip this key lines.append(line) if not lines: continue # no keys for this host contents = "\n".join(lines) + "\n" to = tarfile.TarInfo(name=f) # These will only be used where the username doesn't # exist on the target system for some reason; hence, # in those cases, the safest thing is for the file to # be owned by root but group nobody. This deals with # the bloody obscure case where the group fails to exist # whilst the user does (in which case we want to avoid # ending up with a file which is owned user:root to avoid # a fairly obvious attack vector) to.uid = 0 to.gid = 65534 # Using the username / groupname fields avoids any need # to give a shit^W^W^Wcare about the UIDoffset stuff. to.uname = f to.gname = grname to.mode = 0400 to.mtime = int(time.time()) to.size = len(contents) tf.addfile(to, StringIO(contents)) tf.close() os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target) # add a list of groups to existing groups, # including all subgroups thereof, recursively. # basically this proceduces the transitive hull of the groups in # addgroups. def addGroups(existingGroups, newGroups, uid, current_host): for group in newGroups: # if it's a @host, split it and verify it's on the current host. s = group.split('@', 1) if len(s) == 2 and s[1] != current_host: continue group = s[0] # let's see if we handled this group already if group in existingGroups: continue if not GroupIDMap.has_key(group): print "Group", group, "does not exist but", uid, "is in it" continue existingGroups.append(group) if SubGroupMap.has_key(group): addGroups(existingGroups, SubGroupMap[group], uid, current_host) # Generate the group list def GenGroup(accounts, File, current_host): grouprevmap = {} F = None try: F = open(File + ".tdb.tmp", "w") # Generate the GroupMap GroupMap = {} for x in GroupIDMap: GroupMap[x] = [] GroupHasPrimaryMembers = {} # Sort them into a list of groups having a set of users for a in accounts: GroupHasPrimaryMembers[ a['gidNumber'] ] = True if not 'supplementaryGid' in a: continue supgroups=[] addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host) for g in supgroups: GroupMap[g].append(a['uid']) # Output the group file. J = 0 for x in GroupMap.keys(): if not x in GroupIDMap: continue if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers: continue grouprevmap[GroupIDMap[x]] = x Line = "%s:x:%u:" % (x, GroupIDMap[x]) Comma = '' for I in GroupMap[x]: Line = Line + ("%s%s" % (Comma, I)) Comma = ',' Line = Sanitize(Line) + "\n" F.write("0%u %s" % (J, Line)) F.write(".%s %s" % (x, Line)) F.write("=%u %s" % (GroupIDMap[x], Line)) J = J + 1 # Oops, something unspeakable happened. except: Die(File, None, F) raise Done(File, None, F) return grouprevmap def CheckForward(accounts): for a in accounts: if not 'emailForward' in a: continue delete = False # Do not allow people to try to buffer overflow busted parsers if len(a['emailForward']) > 200: delete = True # Check the forwarding address elif EmailCheck.match(a['emailForward']) is None: delete = True if delete: a.delete_mailforward() # Generate the email forwarding list def GenForward(accounts, File): F = None try: OldMask = os.umask(0022) F = open(File + ".tmp", "w", 0644) os.umask(OldMask) for a in accounts: if not 'emailForward' in a: continue Line = "%s: %s" % (a['uid'], a['emailForward']) Line = Sanitize(Line) + "\n" F.write(Line) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) def GenCDB(accounts, File, key): Fdb = None try: OldMask = os.umask(0022) # nothing else does the fsync stuff, so why do it here? prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else '' Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w") os.umask(OldMask) # Write out the email address for each user for a in accounts: if not key in a: continue value = a[key] user = a['uid'] Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value)) Fdb.write("\n") # Oops, something unspeakable happened. except: Fdb.close() raise if Fdb.close() != None: raise "cdbmake gave an error" def GenDBM(accounts, File, key): Fdb = None OldMask = os.umask(0022) fn = os.path.join(File).encode('ascii', 'ignore') try: posix.remove(fn) except: pass try: Fdb = dbm.open(fn, "c") os.umask(OldMask) # Write out the email address for each user for a in accounts: if not key in a: continue value = a[key] user = a['uid'] Fdb[user] = value Fdb.close() except: # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db os.remove(File + ".db") raise # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db os.rename (File + ".db", File) # Generate the anon XEarth marker file def GenMarkers(accounts, File): F = None try: F = open(File + ".tmp", "w") # Write out the position for each user for a in accounts: if not ('latitude' in a and 'longitude' in a): continue try: Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True)) Line = Sanitize(Line) + "\n" F.write(Line) except: pass # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate the debian-private subscription list def GenPrivate(accounts, File): F = None try: F = open(File + ".tmp", "w") # Write out the position for each user for a in accounts: if not a.is_active_user(): continue if a.is_guest_account(): continue if not 'privateSub' in a: continue try: Line = "%s"%(a['privateSub']) Line = Sanitize(Line) + "\n" F.write(Line) except: pass # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate a list of locked accounts def GenDisabledAccounts(accounts, File): F = None try: F = open(File + ".tmp", "w") disabled_accounts = [] # Fetch all the users for a in accounts: if a.pw_active(): continue Line = "%s:%s" % (a['uid'], "Account is locked") disabled_accounts.append(a) F.write(Sanitize(Line) + "\n") # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) return disabled_accounts # Generate the list of local addresses that refuse all mail def GenMailDisable(accounts, File): F = None try: F = open(File + ".tmp", "w") for a in accounts: if not 'mailDisableMessage' in a: continue Line = "%s: %s"%(a['uid'], a['mailDisableMessage']) Line = Sanitize(Line) + "\n" F.write(Line) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate a list of uids that should have boolean affects applied def GenMailBool(accounts, File, key): F = None try: F = open(File + ".tmp", "w") for a in accounts: if not key in a: continue if not a[key] == 'TRUE': continue Line = "%s"%(a['uid']) Line = Sanitize(Line) + "\n" F.write(Line) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate a list of hosts for RBL or whitelist purposes. def GenMailList(accounts, File, key): F = None try: F = open(File + ".tmp", "w") if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$') else: validregex = re.compile('^[-\w.]+$') for a in accounts: if not key in a: continue filtered = filter(lambda z: validregex.match(z), a[key]) if len(filtered) == 0: continue if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered) line = a['uid'] + ': ' + ' : '.join(filtered) line = Sanitize(line) + "\n" F.write(line) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) def isRoleAccount(account): return 'debianRoleAccount' in account['objectClass'] # Generate the DNS Zone file def GenDNS(accounts, File): F = None try: F = open(File + ".tmp", "w") # Fetch all the users RRs = {} # Write out the zone file entry for each user for a in accounts: if not 'dnsZoneEntry' in a: continue if not a.is_active_user() and not isRoleAccount(a): continue if a.is_guest_account(): continue try: F.write("; %s\n"%(a.email_address())) for z in a["dnsZoneEntry"]: Split = z.lower().split() if Split[1].lower() == 'in': Line = " ".join(Split) + "\n" F.write(Line) Host = Split[0] + DNSZone if BSMTPCheck.match(Line) != None: F.write("; Has BSMTP\n") # Write some identification information if not RRs.has_key(Host): if Split[2].lower() in ["a", "aaaa"]: Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address()) for y in a["keyFingerPrint"]: Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y)) F.write(Line) RRs[Host] = 1 else: Line = "; Err %s"%(str(Split)) F.write(Line) F.write("\n") except Exception, e: F.write("; Errors:\n") for line in str(e).split("\n"): F.write("; %s\n"%(line)) pass # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) def is_ipv6_addr(i): try: socket.inet_pton(socket.AF_INET6, i) except socket.error: return False return True def ExtractDNSInfo(x): hostname = GetAttr(x, "hostname") TTLprefix="\t" if 'dnsTTL' in x[1]: TTLprefix="%s\t"%(x[1]["dnsTTL"][0]) DNSInfo = [] if x[1].has_key("ipHostNumber"): for I in x[1]["ipHostNumber"]: if is_ipv6_addr(I): DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I)) else: DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I)) Algorithm = None ssh_hostnames = [ hostname ] if x[1].has_key("sshfpHostname"): ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ] if 'sshRSAHostKey' in x[1]: for I in x[1]["sshRSAHostKey"]: Split = I.split() key_prefix = Split[0] key = base64.decodestring(Split[1]) # RFC4255 # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml if key_prefix == 'ssh-rsa': Algorithm = 1 if key_prefix == 'ssh-dss': Algorithm = 2 if key_prefix == 'ssh-ed25519': Algorithm = 4 if Algorithm == None: continue # and more from the registry sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ] fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ] for h in ssh_hostnames: for digest_codepoint, fingerprint in fingerprints: DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint)) if 'architecture' in x[1]: Arch = GetAttr(x, "architecture") Mach = "" if x[1].has_key("machine"): Mach = " " + GetAttr(x, "machine") DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian")) if x[1].has_key("mXRecord"): for I in x[1]["mXRecord"]: if I in MX_remap: for e in MX_remap[I]: DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e)) else: DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I)) return DNSInfo # Generate the DNS records def GenZoneRecords(host_attrs, File): F = None try: F = open(File + ".tmp", "w") # Fetch all the hosts for x in host_attrs: if x[1].has_key("hostname") == 0: continue if IsDebianHost.match(GetAttr(x, "hostname")) is None: continue for Line in ExtractDNSInfo(x): F.write(Line + "\n") # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate the BSMTP file def GenBSMTP(accounts, File, HomePrefix): F = None try: F = open(File + ".tmp", "w") # Write out the zone file entry for each user for a in accounts: if not 'dnsZoneEntry' in a: continue if not a.is_active_user(): continue try: for z in a["dnsZoneEntry"]: Split = z.lower().split() if Split[1].lower() == 'in': for y in range(0, len(Split)): if Split[y] == "$": Split[y] = "\n\t" Line = " ".join(Split) + "\n" Host = Split[0] + DNSZone if BSMTPCheck.match(Line) != None: F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host, a['uid'], HomePrefix, a['uid'], Host)) except: F.write("; Errors\n") pass # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) def HostToIP(Host, mapped=True): IPAdresses = [] if Host[1].has_key("ipHostNumber"): for addr in Host[1]["ipHostNumber"]: IPAdresses.append(addr) if not is_ipv6_addr(addr) and mapped == "True": IPAdresses.append("::ffff:"+addr) return IPAdresses # Generate the ssh known hosts file def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None): F = None try: OldMask = os.umask(0022) F = open(File + ".tmp", "w", 0644) os.umask(OldMask) for x in host_attrs: if x[1].has_key("hostname") == 0 or \ x[1].has_key("sshRSAHostKey") == 0: continue Host = GetAttr(x, "hostname") HostNames = [ Host ] if Host.endswith(HostDomain): HostNames.append(Host[:-(len(HostDomain) + 1)]) # in the purpose field [[host|some other text]] (where some other text is optional) # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts # file. But so that we don't have to add everything we link we can add an asterisk # and say [[*... to ignore it. In order to be able to add stuff to ssh without # http linking it we also support [[-hostname]] entries. for i in x[1].get("purpose", []): m = PurposeHostField.match(i) if m: m = m.group(1) # we ignore [[*..]] entries if m.startswith('*'): continue if m.startswith('-'): m = m[1:] if m: HostNames.append(m) if m.endswith(HostDomain): HostNames.append(m[:-(len(HostDomain) + 1)]) for I in x[1]["sshRSAHostKey"]: if mode and mode == 'authorized_keys': hosts = HostToIP(x) if 'sshdistAuthKeysHost' in x[1]: hosts += x[1]['sshdistAuthKeysHost'] clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host) clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand) Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I) else: Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I) Line = Sanitize(Line) + "\n" F.write(Line) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) # Generate the debianhosts file (list of all IP addresses) def GenHosts(host_attrs, File): F = None try: OldMask = os.umask(0022) F = open(File + ".tmp", "w", 0644) os.umask(OldMask) seen = set() for x in host_attrs: if IsDebianHost.match(GetAttr(x, "hostname")) is None: continue if not 'ipHostNumber' in x[1]: continue addrs = x[1]["ipHostNumber"] for addr in addrs: if addr not in seen: seen.add(addr) addr = Sanitize(addr) + "\n" F.write(addr) # Oops, something unspeakable happened. except: Die(File, F, None) raise Done(File, F, None) def replaceTree(src, dst_basedir): bn = os.path.basename(src) dst = os.path.join(dst_basedir, bn) safe_rmtree(dst) shutil.copytree(src, dst) def GenKeyrings(OutDir): for k in Keyrings: if os.path.isdir(k): replaceTree(k, OutDir) else: shutil.copy(k, OutDir) def get_accounts(ldap_conn): # Fetch all the users passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\ ["uid", "uidNumber", "gidNumber", "supplementaryGid",\ "gecos", "loginShell", "userPassword", "shadowLastChange",\ "shadowMin", "shadowMax", "shadowWarning", "shadowInactive", "shadowExpire", "emailForward", "latitude", "longitude",\ "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\ "keyFingerPrint", "privateSub", "mailDisableMessage",\ "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ "mailContentInspectionAction", "webPassword", "rtcPassword",\ "bATVToken", "totpSeed"]) if passwd_attrs is None: raise UDEmptyList, "No Users" accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs) accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower())) return accounts def get_hosts(ldap_conn): # Fetch all the hosts HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\ ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\ "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture", "sshfpHostname"]) if HostAttrs == None: raise UDEmptyList, "No Hosts" HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower())) return HostAttrs def make_ldap_conn(): # Connect to the ldap server l = connectLDAP() # for testing purposes it's sometimes useful to pass username/password # via the environment if 'UD_CREDENTIALS' in os.environ: Pass = os.environ['UD_CREDENTIALS'].split() else: F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r") Pass = F.readline().strip().split(" ") F.close() l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1]) return l def setup_group_maps(l): # Fetch all the groups group_id_map = {} subgroup_map = {} attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\ ["gid", "gidNumber", "subGroup"]) # Generate the subgroup_map and group_id_map for x in attrs: if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled": continue if x[1].has_key("gidNumber") == 0: continue group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]) if x[1].has_key("subGroup") != 0: subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"]) global SubGroupMap global GroupIDMap SubGroupMap = subgroup_map GroupIDMap = group_id_map def generate_all(global_dir, ldap_conn): accounts = get_accounts(ldap_conn) host_attrs = get_hosts(ldap_conn) global_dir += '/' # Generate global things accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts") accounts = filter(lambda x: not IsRetired(x), accounts) CheckForward(accounts) GenMailDisable(accounts, global_dir + "mail-disable") GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward') GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward') GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction') GenPrivate(accounts, global_dir + "debian-private") GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock') GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting") GenMailBool(accounts, global_dir + "mail-callout", "mailCallout") GenMailList(accounts, global_dir + "mail-rbl", "mailRBL") GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL") GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist") GenWebPassword(accounts, global_dir + "web-passwords") GenRtcPassword(accounts, global_dir + "rtc-passwords") GenTOTPSeed(accounts, global_dir + "users.oath") GenKeyrings(global_dir) # Compatibility. GenForward(accounts, global_dir + "forward-alias") GenAllUsers(accounts, global_dir + 'all-accounts.json') accounts = filter(lambda a: not a in accounts_disabled, accounts) ssh_userkeys = GenSSHShadow(global_dir, accounts) GenMarkers(accounts, global_dir + "markers") GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts") GenHosts(host_attrs, global_dir + "debianhosts") GenDNS(accounts, global_dir + "dns-zone") GenZoneRecords(host_attrs, global_dir + "dns-sshfp") setup_group_maps(ldap_conn) for host in host_attrs: if not "hostname" in host[1]: continue generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys) def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): current_host = host[1]['hostname'][0] OutDir = global_dir + current_host + '/' if not os.path.isdir(OutDir): os.mkdir(OutDir) # Get the group list and convert any named groups to numerics GroupList = {} for groupname in AllowedGroupsPreload.strip().split(" "): GroupList[groupname] = True if 'allowedGroups' in host[1]: for groupname in host[1]['allowedGroups']: GroupList[groupname] = True for groupname in GroupList.keys(): if groupname in GroupIDMap: GroupList[str(GroupIDMap[groupname])] = True ExtraList = {} if 'exportOptions' in host[1]: for extra in host[1]['exportOptions']: ExtraList[extra.upper()] = True if GroupList != {}: accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts) DoLink(global_dir, OutDir, "debianhosts") DoLink(global_dir, OutDir, "ssh_known_hosts") DoLink(global_dir, OutDir, "disabled-accounts") sys.stdout.flush() if 'NOPASSWD' in ExtraList: userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*") else: userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x") sys.stdout.flush() grouprevmap = GenGroup(accounts, OutDir + "group", current_host) GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host) # Now we know who we're allowing on the machine, export # the relevant ssh keys GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host) if not 'NOPASSWD' in ExtraList: GenShadow(accounts, OutDir + "shadow") # Link in global things if not 'NOMARKERS' in ExtraList: DoLink(global_dir, OutDir, "markers") DoLink(global_dir, OutDir, "mail-forward.cdb") DoLink(global_dir, OutDir, "mail-forward.db") DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb") DoLink(global_dir, OutDir, "mail-contentinspectionaction.db") DoLink(global_dir, OutDir, "mail-disable") DoLink(global_dir, OutDir, "mail-greylist") DoLink(global_dir, OutDir, "mail-callout") DoLink(global_dir, OutDir, "mail-rbl") DoLink(global_dir, OutDir, "mail-rhsbl") DoLink(global_dir, OutDir, "mail-whitelist") DoLink(global_dir, OutDir, "all-accounts.json") GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward') GenDBM(accounts, OutDir + "user-forward.db", 'emailForward') GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken') GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken') GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions') GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions') # Compatibility. DoLink(global_dir, OutDir, "forward-alias") if 'DNS' in ExtraList: DoLink(global_dir, OutDir, "dns-zone") DoLink(global_dir, OutDir, "dns-sshfp") if 'AUTHKEYS' in ExtraList: DoLink(global_dir, OutDir, "authorized_keys") if 'BSMTP' in ExtraList: GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix) if 'PRIVATE' in ExtraList: DoLink(global_dir, OutDir, "debian-private") if 'GITOLITE' in ExtraList: GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host) if 'exportOptions' in host[1]: for entry in host[1]['exportOptions']: v = entry.split('=',1) if v[0] != 'GITOLITE' or len(v) != 2: continue options = v[1].split(',') group = options.pop(0); gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts) if not 'nohosts' in options: gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts) else: gitolite_hosts = [] command = None for opt in options: if opt.startswith('sshcmd='): command = opt.split('=',1)[1] GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host) if 'WEB-PASSWORDS' in ExtraList: DoLink(global_dir, OutDir, "web-passwords") if 'RTC-PASSWORDS' in ExtraList: DoLink(global_dir, OutDir, "rtc-passwords") if 'TOTP' in ExtraList: DoLink(global_dir, OutDir, "users.oath") if 'KEYRING' in ExtraList: for k in Keyrings: bn = os.path.basename(k) if os.path.isdir(k): src = os.path.join(global_dir, bn) replaceTree(src, OutDir) else: DoLink(global_dir, OutDir, bn) else: for k in Keyrings: try: bn = os.path.basename(k) target = os.path.join(OutDir, bn) if os.path.isdir(target): safe_rmtree(dst) else: posix.remove(target) except: pass DoLink(global_dir, OutDir, "last_update.trace") def getLastLDAPChangeTime(l): mods = l.search_s('cn=log', ldap.SCOPE_ONELEVEL, '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))', ['reqEnd']) last = 0 # Sort the list by reqEnd sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0]) # Take the last element in the array last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0] return last def getLastKeyringChangeTime(): krmod = 0 for k in Keyrings: mt = os.path.getmtime(k) if mt > krmod: krmod = mt return int(krmod) def getLastBuildTime(gdir): cache_last_ldap_mod = 0 cache_last_unix_mod = 0 cache_last_run = 0 try: fd = open(os.path.join(gdir, "last_update.trace"), "r") cache_last_mod=fd.read().split() try: cache_last_ldap_mod = cache_last_mod[0] cache_last_unix_mod = int(cache_last_mod[1]) cache_last_run = int(cache_last_mod[2]) except IndexError, ValueError: pass fd.close() except IOError, e: if e.errno == errno.ENOENT: pass else: raise e return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run) def mq_notify(options, message): options.section = 'dsa-udgenerate' options.config = '/etc/dsa/pubsub.conf' config = Config(options) conf = { 'rabbit_userid': config.username, 'rabbit_password': config.password, 'rabbit_virtual_host': config.vhost, 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'], 'use_ssl': False } msg = { 'message': message, 'timestamp': int(time.time()) } conn = None try: conn = Connection(conf=conf) conn.topic_send(config.topic, json.dumps(msg), exchange_name=config.exchange, timeout=5) finally: if conn: conn.close() def ud_generate(): parser = optparse.OptionParser() parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR", help="Output directory.") parser.add_option("-f", "--force", dest="force", action="store_true", help="Force generation, even if no update to LDAP has happened.") (options, args) = parser.parse_args() if len(args) > 0: parser.print_help() sys.exit(1) if options.generatedir is not None: generate_dir = os.environ['UD_GENERATEDIR'] elif 'UD_GENERATEDIR' in os.environ: generate_dir = os.environ['UD_GENERATEDIR'] else: generate_dir = GenerateDir lockf = os.path.join(generate_dir, 'ud-generate.lock') lock = get_lock( lockf ) if lock is None: sys.stderr.write("Could not acquire lock %s.\n"%(lockf)) sys.exit(1) l = make_ldap_conn() time_started = int(time.time()) ldap_last_mod = getLastLDAPChangeTime(l) unix_last_mod = getLastKeyringChangeTime() cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir) need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE) fd = open(os.path.join(generate_dir, "last_update.trace"), "w") if need_update or options.force: msg = 'Update forced' if options.force else 'Update needed' generate_all(generate_dir, l) if use_mq: mq_notify(options, msg) last_run = int(time.time()) fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run)) fd.close() sys.exit(0) if __name__ == "__main__": if 'UD_PROFILE' in os.environ: import cProfile import pstats cProfile.run('ud_generate()', "udg_prof") p = pstats.Stats('udg_prof') ##p.sort_stats('time').print_stats() p.sort_stats('cumulative').print_stats() else: ud_generate() # vim:set et: # vim:set ts=3: # vim:set shiftwidth=3: