X-Git-Url: https://git.adam-barratt.org.uk/?p=mirror%2Fuserdir-ldap.git;a=blobdiff_plain;f=ud-generate;h=a787dfad5df514b5f7120ac6d5e92767e32d64c0;hp=5529d93a312b93d899aa177ac87a1c035b99c5c7;hb=HEAD;hpb=9b17f1fdeb031eb57906422dd4615d5a31f0f8a4 diff --git a/ud-generate b/ud-generate index 5529d93..a787dfa 100755 --- a/ud-generate +++ b/ud-generate @@ -6,7 +6,7 @@ # Copyright (c) 2003-2004 James Troup # Copyright (c) 2004-2005,7 Joey Schulze # Copyright (c) 2001-2007 Ryan Murray -# Copyright (c) 2008,2009,2010 Peter Palfrader +# Copyright (c) 2008,2009,2010,2011 Peter Palfrader # Copyright (c) 2008 Andreas Barth # Copyright (c) 2008 Mark Hymers # Copyright (c) 2008 Luk Claes @@ -28,10 +28,16 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import string, re, time, ldap, getopt, sys, os, pwd, posix, socket, base64, sha, shutil, errno, tarfile, grp +from dsa_mq.connection import Connection +from dsa_mq.config import Config + +import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm from userdir_ldap import * from userdir_exceptions import * import UDLdap +from xml.etree.ElementTree import Element, SubElement, Comment +from xml.etree import ElementTree +from xml.dom import minidom try: from cStringIO import StringIO except ImportError: @@ -52,23 +58,36 @@ if os.getuid() == 0: # # GLOBAL STATE # -GroupIDMap = {} -SubGroupMap = {} -Allowed = None -CurrentHost = "" +GroupIDMap = None +SubGroupMap = None UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' +MAX_UD_AGE = 3600*24 -EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)?$") +EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$") BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL) PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]") -IsV6Addr = re.compile("^[a-fA-F0-9:]+$") IsDebianHost = re.compile(ConfModule.dns_hostmatch) isSSHFP = re.compile("^\s*IN\s+SSHFP") DNSZone = ".debian.net" Keyrings = ConfModule.sync_keyrings.split(":") +GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None) +GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None) +GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", ".")) +MX_remap = json.loads(ConfModule.MX_remap) +use_mq = getattr(ConfModule, "use_mq", True) + +rtc_realm = getattr(ConfModule, "rtc_realm", None) +rtc_append = getattr(ConfModule, "rtc_append", None) + +def prettify(elem): + """Return a pretty-printed XML string for the Element. + """ + rough_string = ElementTree.tostring(elem, 'utf-8') + reparsed = minidom.parseString(rough_string) + return reparsed.toprettyxml(indent=" ") def safe_makedirs(dir): try: @@ -88,6 +107,25 @@ def safe_rmtree(dir): else: raise e +def get_lock(fn, wait=5*60): + f = open(fn, "w") + sl = 0.1 + ends = time.time() + wait + + while True: + success = False + try: + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return f + except IOError: + pass + if time.time() >= ends: + return None + sl = min(sl*2, 10, ends - time.time()) + time.sleep(sl) + return None + + def Sanitize(Str): return Str.translate(string.maketrans("\n\r\t", "$$$")) @@ -127,27 +165,21 @@ def IsRetired(account): return False -#def IsGidDebian(account): -# return account['gidNumber'] == 800 - # See if this user is in the group list -def IsInGroup(account): - if Allowed is None: - return True - +def IsInGroup(account, allowed, current_host): # See if the primary group is in the list - if str(account['gidNumber']) in Allowed: return True + if str(account['gidNumber']) in allowed: return True # Check the host based ACL - if account.is_allowed_by_hostacl(CurrentHost): return True + if account.is_allowed_by_hostacl(current_host): return True # See if there are supplementary groups if not 'supplementaryGid' in account: return False supgroups=[] - addGroups(supgroups, account['supplementaryGid'], account['uid']) + addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host) for g in supgroups: - if Allowed.has_key(g): + if g in allowed: return True return False @@ -182,8 +214,6 @@ def GenPasswd(accounts, File, HomePrefix, PwdMarker): userlist = {} i = 0 for a in accounts: - if not IsInGroup(a): continue - # Do not let people try to buffer overflow some busted passwd parser. if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue @@ -241,9 +271,6 @@ def GenShadow(accounts, File): i = 0 for a in accounts: - Pass = '*' - if not IsInGroup(a): continue - # If the account is locked, mark it as such in shadow # See Debian Bug #308229 for why we set it to 1 instead of 0 if not a.pw_active(): ShadowExpire = '1' @@ -270,7 +297,7 @@ def GenShadow(accounts, File): Done(File, None, F) # Generate the sudo passwd file -def GenShadowSudo(accounts, File, untrusted): +def GenShadowSudo(accounts, File, untrusted, current_host): F = None try: OldMask = os.umask(0077) @@ -279,11 +306,9 @@ def GenShadowSudo(accounts, File, untrusted): for a in accounts: Pass = '*' - if not IsInGroup(a): continue - if 'sudoPassword' in a: for entry in a['sudoPassword']: - Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry) + Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry) if Match == None: continue uuid = Match.group(1) @@ -294,7 +319,7 @@ def GenShadowSudo(accounts, File, untrusted): if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass): continue for_all = hosts == "*" - for_this_host = CurrentHost in hosts.split(',') + for_this_host = current_host in hosts.split(',') if not (for_all or for_this_host): continue # ignore * passwords for untrusted hosts, but copy host specific passwords @@ -316,48 +341,142 @@ def GenShadowSudo(accounts, File, untrusted): raise Done(File, F, None) +# Generate the sudo passwd file +def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None): + F = None + if sshcommand is None: + sshcommand = GitoliteSSHCommand + try: + OldMask = os.umask(0022) + F = open(File + ".tmp", "w", 0600) + os.umask(OldMask) + + if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "": + for a in accounts: + if not 'sshRSAAuthKey' in a: continue + + User = a['uid'] + prefix = GitoliteSSHRestrictions + prefix = prefix.replace('@@COMMAND@@', sshcommand) + prefix = prefix.replace('@@USER@@', User) + for I in a["sshRSAAuthKey"]: + if I.startswith("allowed_hosts=") and ' ' in line: + if current_host is None: + continue + machines, I = I.split('=', 1)[1].split(' ', 1) + if current_host not in machines.split(','): + continue # skip this key + + if I.startswith('ssh-'): + line = "%s %s"%(prefix, I) + else: + continue # do not allow keys with other restrictions that might conflict + line = Sanitize(line) + "\n" + F.write(line) + + for dn, attrs in hosts: + if not 'sshRSAHostKey' in attrs: continue + hostname = "host-" + attrs['hostname'][0] + prefix = GitoliteSSHRestrictions + prefix = prefix.replace('@@COMMAND@@', sshcommand) + prefix = prefix.replace('@@USER@@', hostname) + for I in attrs["sshRSAHostKey"]: + line = "%s %s"%(prefix, I) + line = Sanitize(line) + "\n" + F.write(line) + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + # Generate the shadow list def GenSSHShadow(global_dir, accounts): # Fetch all the users - userfiles = [] - - safe_rmtree(os.path.join(global_dir, 'userkeys')) - safe_makedirs(os.path.join(global_dir, 'userkeys')) + userkeys = {} for a in accounts: if not 'sshRSAAuthKey' in a: continue - F = None - try: - OldMask = os.umask(0077) - File = os.path.join(global_dir, 'userkeys', a['uid']) - F = open(File + ".tmp", "w", 0600) - os.umask(OldMask) + contents = [] + for I in a['sshRSAAuthKey']: + MultipleLine = "%s" % I + MultipleLine = Sanitize(MultipleLine) + contents.append(MultipleLine) + userkeys[a['uid']] = contents + return userkeys - for I in a['sshRSAAuthKey']: - MultipleLine = "%s" % I - MultipleLine = Sanitize(MultipleLine) + "\n" - F.write(MultipleLine) +# Generate the webPassword list +def GenWebPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) - Done(File, F, None) - userfiles.append(os.path.basename(File)) + for a in accounts: + if not 'webPassword' in a: continue + if not a.pw_active(): continue - # Oops, something unspeakable happened. - except IOError: - Die(File, F, None) - # As neither masterFileName nor masterFile are defined at any point - # this will raise a NameError. - Die(masterFileName, masterFile, None) - raise + Pass = str(a['webPassword']) + Line = "%s:%s" % (a['uid'], Pass) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) - return userfiles + except: + Die(File, None, F) + raise + +# Generate the rtcPassword list +def GenRtcPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + for a in accounts: + if a.is_guest_account(): continue + if not 'rtcPassword' in a: continue + if not a.pw_active(): continue + + Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + + except: + Die(File, None, F) + raise + +# Generate the TOTP auth file +def GenTOTPSeed(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + F.write("# Option User Prefix Seed\n") + for a in accounts: + if a.is_guest_account(): continue + if not 'totpSeed' in a: continue + if not a.pw_active(): continue -def GenSSHtarballs(global_dir, userlist, SSHFiles, grouprevmap, target): + Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed']) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + except: + Die(File, None, F) + raise + + +def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host): OldMask = os.umask(0077) - tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % CurrentHost), mode='w:gz') + tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz') os.umask(OldMask) - for f in userlist.keys(): - if f not in SSHFiles: + for f in userlist: + if f not in ssh_userkeys: continue # If we're not exporting their primary group, don't export # the key and warn @@ -375,10 +494,21 @@ def GenSSHtarballs(global_dir, userlist, SSHFiles, grouprevmap, target): pass if grname is None: - print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, CurrentHost, userlist[f]) + print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f]) continue - to = tf.gettarinfo(os.path.join(global_dir, 'userkeys', f), f) + lines = [] + for line in ssh_userkeys[f]: + if line.startswith("allowed_hosts=") and ' ' in line: + machines, line = line.split('=', 1)[1].split(' ', 1) + if current_host not in machines.split(','): + continue # skip this key + lines.append(line) + if not lines: + continue # no keys for this host + contents = "\n".join(lines) + "\n" + + to = tarfile.TarInfo(name=f) # These will only be used where the username doesn't # exist on the target system for some reason; hence, # in those cases, the safest thing is for the file to @@ -394,33 +524,23 @@ def GenSSHtarballs(global_dir, userlist, SSHFiles, grouprevmap, target): to.uname = f to.gname = grname to.mode = 0400 - - contents = file(os.path.join(global_dir, 'userkeys', f)).read() - lines = [] - for line in contents.splitlines(): - if line.startswith("allowed_hosts=") and ' ' in line: - machines, line = line.split('=', 1)[1].split(' ', 1) - if CurrentHost not in machines.split(','): - continue # skip this key - lines.append(line) - if not lines: - continue # no keys for this host - contents = "\n".join(lines) + "\n" + to.mtime = int(time.time()) to.size = len(contents) + tf.addfile(to, StringIO(contents)) tf.close() - os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % CurrentHost), target) + os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target) # add a list of groups to existing groups, # including all subgroups thereof, recursively. # basically this proceduces the transitive hull of the groups in # addgroups. -def addGroups(existingGroups, newGroups, uid): +def addGroups(existingGroups, newGroups, uid, current_host): for group in newGroups: # if it's a @host, split it and verify it's on the current host. s = group.split('@', 1) - if len(s) == 2 and s[1] != CurrentHost: + if len(s) == 2 and s[1] != current_host: continue group = s[0] @@ -435,10 +555,10 @@ def addGroups(existingGroups, newGroups, uid): existingGroups.append(group) if SubGroupMap.has_key(group): - addGroups(existingGroups, SubGroupMap[group], uid) + addGroups(existingGroups, SubGroupMap[group], uid, current_host) # Generate the group list -def GenGroup(accounts, File): +def GenGroup(accounts, File, current_host): grouprevmap = {} F = None try: @@ -446,25 +566,24 @@ def GenGroup(accounts, File): # Generate the GroupMap GroupMap = {} - for x in GroupIDMap.keys(): + for x in GroupIDMap: GroupMap[x] = [] GroupHasPrimaryMembers = {} # Sort them into a list of groups having a set of users for a in accounts: GroupHasPrimaryMembers[ a['gidNumber'] ] = True - if not IsInGroup(a): continue if not 'supplementaryGid' in a: continue supgroups=[] - addGroups(supgroups, a['supplementaryGid'], a['uid']) + addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host) for g in supgroups: GroupMap[g].append(a['uid']) # Output the group file. J = 0 for x in GroupMap.keys(): - if GroupIDMap.has_key(x) == 0: + if not x in GroupIDMap: continue if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers: @@ -495,12 +614,10 @@ def CheckForward(accounts): for a in accounts: if not 'emailForward' in a: continue - delete = False - if not IsInGroup(a): delete = True # Do not allow people to try to buffer overflow busted parsers - elif len(a['emailForward']) > 200: delete = True + if len(a['emailForward']) > 200: delete = True # Check the forwarding address elif EmailCheck.match(a['emailForward']) is None: delete = True @@ -531,7 +648,9 @@ def GenCDB(accounts, File, key): Fdb = None try: OldMask = os.umask(0022) - Fdb = os.popen("cdbmake %s %s.tmp"%(File, File), "w") + # nothing else does the fsync stuff, so why do it here? + prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else '' + Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w") os.umask(OldMask) # Write out the email address for each user @@ -549,6 +668,34 @@ def GenCDB(accounts, File, key): if Fdb.close() != None: raise "cdbmake gave an error" +def GenDBM(accounts, File, key): + Fdb = None + OldMask = os.umask(0022) + fn = os.path.join(File).encode('ascii', 'ignore') + try: + posix.remove(fn) + except: + pass + + try: + Fdb = dbm.open(fn, "c") + os.umask(OldMask) + + # Write out the email address for each user + for a in accounts: + if not key in a: continue + value = a[key] + user = a['uid'] + Fdb[user] = value + + Fdb.close() + except: + # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db + os.remove(File + ".db") + raise + # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db + os.rename (File + ".db", File) + # Generate the anon XEarth marker file def GenMarkers(accounts, File): F = None @@ -580,6 +727,7 @@ def GenPrivate(accounts, File): # Write out the position for each user for a in accounts: if not a.is_active_user(): continue + if a.is_guest_account(): continue if not 'privateSub' in a: continue try: Line = "%s"%(a['privateSub']) @@ -693,22 +841,20 @@ def GenDNS(accounts, File): for a in accounts: if not 'dnsZoneEntry' in a: continue if not a.is_active_user() and not isRoleAccount(a): continue + if a.is_guest_account(): continue try: F.write("; %s\n"%(a.email_address())) for z in a["dnsZoneEntry"]: Split = z.lower().split() if Split[1].lower() == 'in': - for y in range(0, len(Split)): - if Split[y] == "$": - Split[y] = "\n\t" Line = " ".join(Split) + "\n" F.write(Line) - + Host = Split[0] + DNSZone if BSMTPCheck.match(Line) != None: F.write("; Has BSMTP\n") - + # Write some identification information if not RRs.has_key(Host): if Split[2].lower() in ["a", "aaaa"]: @@ -720,7 +866,7 @@ def GenDNS(accounts, File): else: Line = "; Err %s"%(str(Split)) F.write(Line) - + F.write("\n") except Exception, e: F.write("; Errors:\n") @@ -734,7 +880,15 @@ def GenDNS(accounts, File): raise Done(File, F, None) +def is_ipv6_addr(i): + try: + socket.inet_pton(socket.AF_INET6, i) + except socket.error: + return False + return True + def ExtractDNSInfo(x): + hostname = GetAttr(x, "hostname") TTLprefix="\t" if 'dnsTTL' in x[1]: @@ -743,35 +897,55 @@ def ExtractDNSInfo(x): DNSInfo = [] if x[1].has_key("ipHostNumber"): for I in x[1]["ipHostNumber"]: - if IsV6Addr.match(I) != None: - DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I)) + if is_ipv6_addr(I): + DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I)) else: - DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I)) + DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I)) Algorithm = None + ssh_hostnames = [ hostname ] + if x[1].has_key("sshfpHostname"): + ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ] + if 'sshRSAHostKey' in x[1]: for I in x[1]["sshRSAHostKey"]: Split = I.split() - if Split[0] == 'ssh-rsa': + key_prefix = Split[0] + key = base64.decodestring(Split[1]) + + # RFC4255 + # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml + if key_prefix == 'ssh-rsa': Algorithm = 1 - if Split[0] == 'ssh-dss': + if key_prefix == 'ssh-dss': Algorithm = 2 + if key_prefix == 'ssh-ed25519': + Algorithm = 4 if Algorithm == None: continue - Fingerprint = sha.new(base64.decodestring(Split[1])).hexdigest() - DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint)) + # and more from the registry + sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ] + + fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ] + for h in ssh_hostnames: + for digest_codepoint, fingerprint in fingerprints: + DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint)) if 'architecture' in x[1]: Arch = GetAttr(x, "architecture") Mach = "" if x[1].has_key("machine"): Mach = " " + GetAttr(x, "machine") - DNSInfo.append("%sIN\tHINFO\t\"%s%s\" \"%s\"" % (TTLprefix, Arch, Mach, "Debian GNU/Linux")) + DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian")) if x[1].has_key("mXRecord"): for I in x[1]["mXRecord"]: - DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I)) + if I in MX_remap: + for e in MX_remap[I]: + DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e)) + else: + DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I)) return DNSInfo @@ -789,40 +963,9 @@ def GenZoneRecords(host_attrs, File): if IsDebianHost.match(GetAttr(x, "hostname")) is None: continue - DNSInfo = ExtractDNSInfo(x) - start = True - for Line in DNSInfo: - if start == True: - Line = "%s.\t%s" % (GetAttr(x, "hostname"), Line) - start = False - else: - Line = "\t\t\t%s" % (Line) - + for Line in ExtractDNSInfo(x): F.write(Line + "\n") - # this would write sshfp lines for services on machines - # but we can't yet, since some are cnames and we'll make - # an invalid zonefile - # - # for i in x[1].get("purpose", []): - # m = PurposeHostField.match(i) - # if m: - # m = m.group(1) - # # we ignore [[*..]] entries - # if m.startswith('*'): - # continue - # if m.startswith('-'): - # m = m[1:] - # if m: - # if not m.endswith(HostDomain): - # continue - # if not m.endswith('.'): - # m = m + "." - # for Line in DNSInfo: - # if isSSHFP.match(Line): - # Line = "%s\t%s" % (m, Line) - # F.write(Line + "\n") - # Oops, something unspeakable happened. except: Die(File, F, None) @@ -871,13 +1014,13 @@ def HostToIP(Host, mapped=True): if Host[1].has_key("ipHostNumber"): for addr in Host[1]["ipHostNumber"]: IPAdresses.append(addr) - if IsV6Addr.match(addr) is None and mapped == "True": + if not is_ipv6_addr(addr) and mapped == "True": IPAdresses.append("::ffff:"+addr) return IPAdresses # Generate the ssh known hosts file -def GenSSHKnown(host_attrs, File, mode=None): +def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None): F = None try: OldMask = os.umask(0022) @@ -917,7 +1060,9 @@ def GenSSHKnown(host_attrs, File, mode=None): hosts = HostToIP(x) if 'sshdistAuthKeysHost' in x[1]: hosts += x[1]['sshdistAuthKeysHost'] - Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (Host, ",".join(hosts), I) + clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host) + clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand) + Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I) else: Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I) Line = Sanitize(Line) + "\n" @@ -959,14 +1104,23 @@ def GenHosts(host_attrs, File): raise Done(File, F, None) +def replaceTree(src, dst_basedir): + bn = os.path.basename(src) + dst = os.path.join(dst_basedir, bn) + safe_rmtree(dst) + shutil.copytree(src, dst) + def GenKeyrings(OutDir): for k in Keyrings: - shutil.copy(k, OutDir) + if os.path.isdir(k): + replaceTree(k, OutDir) + else: + shutil.copy(k, OutDir) def get_accounts(ldap_conn): # Fetch all the users - passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0)))",\ + passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\ ["uid", "uidNumber", "gidNumber", "supplementaryGid",\ "gecos", "loginShell", "userPassword", "shadowLastChange",\ "shadowMin", "shadowMax", "shadowWarning", "shadowInactive", @@ -975,7 +1129,8 @@ def get_accounts(ldap_conn): "keyFingerPrint", "privateSub", "mailDisableMessage",\ "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ - "mailContentInspectionAction"]) + "mailContentInspectionAction", "webPassword", "rtcPassword",\ + "bATVToken", "totpSeed"]) if passwd_attrs is None: raise UDEmptyList, "No Users" @@ -988,7 +1143,8 @@ def get_hosts(ldap_conn): # Fetch all the hosts HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\ ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\ - "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"]) + "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture", + "sshfpHostname"]) if HostAttrs == None: raise UDEmptyList, "No Hosts" @@ -1013,6 +1169,30 @@ def make_ldap_conn(): return l + + +def setup_group_maps(l): + # Fetch all the groups + group_id_map = {} + subgroup_map = {} + attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\ + ["gid", "gidNumber", "subGroup"]) + + # Generate the subgroup_map and group_id_map + for x in attrs: + if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled": + continue + if x[1].has_key("gidNumber") == 0: + continue + group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]) + if x[1].has_key("subGroup") != 0: + subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"]) + + global SubGroupMap + global GroupIDMap + SubGroupMap = subgroup_map + GroupIDMap = group_id_map + def generate_all(global_dir, ldap_conn): accounts = get_accounts(ldap_conn) host_attrs = get_hosts(ldap_conn) @@ -1022,20 +1202,24 @@ def generate_all(global_dir, ldap_conn): accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts") accounts = filter(lambda x: not IsRetired(x), accounts) - #accounts_DDs = filter(lambda x: IsGidDebian(x), accounts) CheckForward(accounts) GenMailDisable(accounts, global_dir + "mail-disable") GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward') + GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward') GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') + GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction') GenPrivate(accounts, global_dir + "debian-private") - GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys') + GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock') GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting") GenMailBool(accounts, global_dir + "mail-callout", "mailCallout") GenMailList(accounts, global_dir + "mail-rbl", "mailRBL") GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL") GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist") + GenWebPassword(accounts, global_dir + "web-passwords") + GenRtcPassword(accounts, global_dir + "rtc-passwords") + GenTOTPSeed(accounts, global_dir + "users.oath") GenKeyrings(global_dir) # Compatibility. @@ -1044,7 +1228,7 @@ def generate_all(global_dir, ldap_conn): GenAllUsers(accounts, global_dir + 'all-accounts.json') accounts = filter(lambda a: not a in accounts_disabled, accounts) - ssh_files = GenSSHShadow(global_dir, accounts) + ssh_userkeys = GenSSHShadow(global_dir, accounts) GenMarkers(accounts, global_dir + "markers") GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts") GenHosts(host_attrs, global_dir + "debianhosts") @@ -1052,20 +1236,18 @@ def generate_all(global_dir, ldap_conn): GenDNS(accounts, global_dir + "dns-zone") GenZoneRecords(host_attrs, global_dir + "dns-sshfp") + setup_group_maps(ldap_conn) + for host in host_attrs: if not "hostname" in host[1]: continue - generate_host(host, global_dir, accounts, ssh_files) - -def generate_host(host, global_dir, accounts, ssh_files): - global CurrentHost + generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys) - CurrentHost = host[1]['hostname'][0] - OutDir = global_dir + CurrentHost + '/' - try: +def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): + current_host = host[1]['hostname'][0] + OutDir = global_dir + current_host + '/' + if not os.path.isdir(OutDir): os.mkdir(OutDir) - except: - pass # Get the group list and convert any named groups to numerics GroupList = {} @@ -1083,10 +1265,8 @@ def generate_host(host, global_dir, accounts, ssh_files): for extra in host[1]['exportOptions']: ExtraList[extra.upper()] = True - global Allowed - Allowed = GroupList - if Allowed == {}: - Allowed = None + if GroupList != {}: + accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts) DoLink(global_dir, OutDir, "debianhosts") DoLink(global_dir, OutDir, "ssh_known_hosts") @@ -1098,12 +1278,12 @@ def generate_host(host, global_dir, accounts, ssh_files): else: userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x") sys.stdout.flush() - grouprevmap = GenGroup(accounts, OutDir + "group") - GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList)) + grouprevmap = GenGroup(accounts, OutDir + "group", current_host) + GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host) # Now we know who we're allowing on the machine, export # the relevant ssh keys - GenSSHtarballs(global_dir, userlist, ssh_files, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz')) + GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host) if not 'NOPASSWD' in ExtraList: GenShadow(accounts, OutDir + "shadow") @@ -1112,7 +1292,9 @@ def generate_host(host, global_dir, accounts, ssh_files): if not 'NOMARKERS' in ExtraList: DoLink(global_dir, OutDir, "markers") DoLink(global_dir, OutDir, "mail-forward.cdb") + DoLink(global_dir, OutDir, "mail-forward.db") DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb") + DoLink(global_dir, OutDir, "mail-contentinspectionaction.db") DoLink(global_dir, OutDir, "mail-disable") DoLink(global_dir, OutDir, "mail-greylist") DoLink(global_dir, OutDir, "mail-callout") @@ -1120,9 +1302,12 @@ def generate_host(host, global_dir, accounts, ssh_files): DoLink(global_dir, OutDir, "mail-rhsbl") DoLink(global_dir, OutDir, "mail-whitelist") DoLink(global_dir, OutDir, "all-accounts.json") - GenCDB(filter(lambda x: IsInGroup(x), accounts), OutDir + "user-forward.cdb", 'emailForward') - GenCDB(filter(lambda x: IsInGroup(x), accounts), OutDir + "batv-tokens.cdb", 'bATVToken') - GenCDB(filter(lambda x: IsInGroup(x), accounts), OutDir + "default-mail-options.cdb", 'mailDefaultOptions') + GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward') + GenDBM(accounts, OutDir + "user-forward.db", 'emailForward') + GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken') + GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken') + GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions') + GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions') # Compatibility. DoLink(global_dir, OutDir, "forward-alias") @@ -1140,38 +1325,188 @@ def generate_host(host, global_dir, accounts, ssh_files): if 'PRIVATE' in ExtraList: DoLink(global_dir, OutDir, "debian-private") + if 'GITOLITE' in ExtraList: + GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host) + if 'exportOptions' in host[1]: + for entry in host[1]['exportOptions']: + v = entry.split('=',1) + if v[0] != 'GITOLITE' or len(v) != 2: continue + options = v[1].split(',') + group = options.pop(0); + gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts) + if not 'nohosts' in options: + gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts) + else: + gitolite_hosts = [] + command = None + for opt in options: + if opt.startswith('sshcmd='): + command = opt.split('=',1)[1] + GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host) + + if 'WEB-PASSWORDS' in ExtraList: + DoLink(global_dir, OutDir, "web-passwords") + + if 'RTC-PASSWORDS' in ExtraList: + DoLink(global_dir, OutDir, "rtc-passwords") + + if 'TOTP' in ExtraList: + DoLink(global_dir, OutDir, "users.oath") + if 'KEYRING' in ExtraList: for k in Keyrings: - DoLink(global_dir, OutDir, os.path.basename(k)) + bn = os.path.basename(k) + if os.path.isdir(k): + src = os.path.join(global_dir, bn) + replaceTree(src, OutDir) + else: + DoLink(global_dir, OutDir, bn) else: for k in Keyrings: try: - posix.remove(OutDir + os.path.basename(k)) + bn = os.path.basename(k) + target = os.path.join(OutDir, bn) + if os.path.isdir(target): + safe_rmtree(dst) + else: + posix.remove(target) except: pass + DoLink(global_dir, OutDir, "last_update.trace") + -l = make_ldap_conn() +def getLastLDAPChangeTime(l): + mods = l.search_s('cn=log', + ldap.SCOPE_ONELEVEL, + '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))', + ['reqEnd']) + + last = 0 + + # Sort the list by reqEnd + sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0]) + # Take the last element in the array + last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0] + + return last + +def getLastKeyringChangeTime(): + krmod = 0 + for k in Keyrings: + mt = os.path.getmtime(k) + if mt > krmod: + krmod = mt -# Fetch all the groups -GroupIDMap = {} -attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\ - ["gid", "gidNumber", "subGroup"]) + return int(krmod) -# Generate the SubGroupMap and GroupIDMap -for x in attrs: - if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled": - continue - if x[1].has_key("gidNumber") == 0: - continue - GroupIDMap[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]) - if x[1].has_key("subGroup") != 0: - SubGroupMap.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"]) +def getLastBuildTime(gdir): + cache_last_ldap_mod = 0 + cache_last_unix_mod = 0 + cache_last_run = 0 -# override globaldir for testing -if 'UD_GENERATEDIR' in os.environ: - GenerateDir = os.environ['UD_GENERATEDIR'] + try: + fd = open(os.path.join(gdir, "last_update.trace"), "r") + cache_last_mod=fd.read().split() + try: + cache_last_ldap_mod = cache_last_mod[0] + cache_last_unix_mod = int(cache_last_mod[1]) + cache_last_run = int(cache_last_mod[2]) + except IndexError, ValueError: + pass + fd.close() + except IOError, e: + if e.errno == errno.ENOENT: + pass + else: + raise e -generate_all(GenerateDir, l) + return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run) + +def mq_notify(options, message): + options.section = 'dsa-udgenerate' + options.config = '/etc/dsa/pubsub.conf' + + config = Config(options) + conf = { + 'rabbit_userid': config.username, + 'rabbit_password': config.password, + 'rabbit_virtual_host': config.vhost, + 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'], + 'use_ssl': False + } + + msg = { + 'message': message, + 'timestamp': int(time.time()) + } + conn = None + try: + conn = Connection(conf=conf) + conn.topic_send(config.topic, + json.dumps(msg), + exchange_name=config.exchange, + timeout=5) + finally: + if conn: + conn.close() + +def ud_generate(): + parser = optparse.OptionParser() + parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR", + help="Output directory.") + parser.add_option("-f", "--force", dest="force", action="store_true", + help="Force generation, even if no update to LDAP has happened.") + + (options, args) = parser.parse_args() + if len(args) > 0: + parser.print_help() + sys.exit(1) + + if options.generatedir is not None: + generate_dir = os.environ['UD_GENERATEDIR'] + elif 'UD_GENERATEDIR' in os.environ: + generate_dir = os.environ['UD_GENERATEDIR'] + else: + generate_dir = GenerateDir + + + lockf = os.path.join(generate_dir, 'ud-generate.lock') + lock = get_lock( lockf ) + if lock is None: + sys.stderr.write("Could not acquire lock %s.\n"%(lockf)) + sys.exit(1) + + l = make_ldap_conn() + + time_started = int(time.time()) + ldap_last_mod = getLastLDAPChangeTime(l) + unix_last_mod = getLastKeyringChangeTime() + cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir) + + need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE) + + fd = open(os.path.join(generate_dir, "last_update.trace"), "w") + if need_update or options.force: + msg = 'Update forced' if options.force else 'Update needed' + generate_all(generate_dir, l) + if use_mq: + mq_notify(options, msg) + last_run = int(time.time()) + fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run)) + fd.close() + sys.exit(0) + + +if __name__ == "__main__": + if 'UD_PROFILE' in os.environ: + import cProfile + import pstats + cProfile.run('ud_generate()', "udg_prof") + p = pstats.Stats('udg_prof') + ##p.sort_stats('time').print_stats() + p.sort_stats('cumulative').print_stats() + else: + ud_generate() # vim:set et: # vim:set ts=3: