X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=ud-generate;h=db6770d3d4ac8e16453784f1803d72de67de1529;hb=refs%2Fheads%2Ffordsa;hp=41350d07093a1f31818c1e3e85ae6c4da67531ee;hpb=3a9baa335455ffcfbe195b1e65de4104405d7fab;p=mirror%2Fuserdir-ldap.git diff --git a/ud-generate b/ud-generate index 41350d0..db6770d 100755 --- a/ud-generate +++ b/ud-generate @@ -32,6 +32,7 @@ from dsa_mq.connection import Connection from dsa_mq.config import Config import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm +import subprocess from userdir_ldap import * from userdir_exceptions import * import UDLdap @@ -46,7 +47,7 @@ try: import simplejson as json except ImportError: import json - if not '__author__' in json.__dict__: + if '__author__' not in json.__dict__: sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n") sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n") @@ -77,6 +78,10 @@ GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None) GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None) GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", ".")) MX_remap = json.loads(ConfModule.MX_remap) +use_mq = getattr(ConfModule, "use_mq", True) + +rtc_realm = getattr(ConfModule, "rtc_realm", None) +rtc_append = getattr(ConfModule, "rtc_append", None) def prettify(elem): """Return a pretty-printed XML string for the Element. @@ -161,9 +166,6 @@ def IsRetired(account): return False -#def IsGidDebian(account): -# return account['gidNumber'] == 800 - # See if this user is in the group list def IsInGroup(account, allowed, current_host): # See if the primary group is in the list @@ -173,7 +175,7 @@ def IsInGroup(account, allowed, current_host): if account.is_allowed_by_hostacl(current_host): return True # See if there are supplementary groups - if not 'supplementaryGid' in account: return False + if 'supplementaryGid' not in account: return False supgroups=[] addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host) @@ -183,9 +185,9 @@ def IsInGroup(account, allowed, current_host): return False def Die(File, F, Fdb): - if F != None: + if F is not None: F.close() - if Fdb != None: + if Fdb is not None: Fdb.close() try: os.remove(File + ".tmp") @@ -197,10 +199,10 @@ def Die(File, F, Fdb): pass def Done(File, F, Fdb): - if F != None: + if F is not None: F.close() os.rename(File + ".tmp", File) - if Fdb != None: + if Fdb is not None: Fdb.close() os.rename(File + ".tdb.tmp", File + ".tdb") @@ -213,6 +215,8 @@ def GenPasswd(accounts, File, HomePrefix, PwdMarker): userlist = {} i = 0 for a in accounts: + if 'loginShell' not in a: + continue # Do not let people try to buffer overflow some busted passwd parser. if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue @@ -307,8 +311,8 @@ def GenShadowSudo(accounts, File, untrusted, current_host): Pass = '*' if 'sudoPassword' in a: for entry in a['sudoPassword']: - Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry) - if Match == None: + Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry) + if Match is None: continue uuid = Match.group(1) status = Match.group(2) @@ -352,7 +356,7 @@ def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None): if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "": for a in accounts: - if not 'sshRSAAuthKey' in a: continue + if 'sshRSAAuthKey' not in a: continue User = a['uid'] prefix = GitoliteSSHRestrictions @@ -374,7 +378,7 @@ def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None): F.write(line) for dn, attrs in hosts: - if not 'sshRSAHostKey' in attrs: continue + if 'sshRSAHostKey' not in attrs: continue hostname = "host-" + attrs['hostname'][0] prefix = GitoliteSSHRestrictions prefix = prefix.replace('@@COMMAND@@', sshcommand) @@ -396,7 +400,7 @@ def GenSSHShadow(global_dir, accounts): userkeys = {} for a in accounts: - if not 'sshRSAAuthKey' in a: continue + if 'sshRSAAuthKey' not in a: continue contents = [] for I in a['sshRSAAuthKey']: @@ -415,7 +419,7 @@ def GenWebPassword(accounts, File): os.umask(OldMask) for a in accounts: - if not 'webPassword' in a: continue + if 'webPassword' not in a: continue if not a.pw_active(): continue Pass = str(a['webPassword']) @@ -436,10 +440,11 @@ def GenRtcPassword(accounts, File): os.umask(OldMask) for a in accounts: - if not 'rtcPassword' in a: continue + if a.is_guest_account(): continue + if 'rtcPassword' not in a: continue if not a.pw_active(): continue - Line = "%s@debian.org:%s:rtc.debian.org:AUTHORIZED" % (a['uid'], str(a['rtcPassword'])) + Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm) Line = Sanitize(Line) + "\n" F.write("%s" % (Line)) @@ -447,6 +452,28 @@ def GenRtcPassword(accounts, File): Die(File, None, F) raise +# Generate the TOTP auth file +def GenTOTPSeed(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + F.write("# Option User Prefix Seed\n") + for a in accounts: + if a.is_guest_account(): continue + if 'totpSeed' not in a: continue + if not a.pw_active(): continue + + Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed']) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + except: + Die(File, None, F) + raise + + def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host): OldMask = os.umask(0077) tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz') @@ -549,7 +576,7 @@ def GenGroup(accounts, File, current_host): # Sort them into a list of groups having a set of users for a in accounts: GroupHasPrimaryMembers[ a['gidNumber'] ] = True - if not 'supplementaryGid' in a: continue + if 'supplementaryGid' not in a: continue supgroups=[] addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host) @@ -559,7 +586,7 @@ def GenGroup(accounts, File, current_host): # Output the group file. J = 0 for x in GroupMap.keys(): - if not x in GroupIDMap: + if x not in GroupIDMap: continue if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers: @@ -588,7 +615,7 @@ def GenGroup(accounts, File, current_host): def CheckForward(accounts): for a in accounts: - if not 'emailForward' in a: continue + if 'emailForward' not in a: continue delete = False @@ -609,7 +636,7 @@ def GenForward(accounts, File): os.umask(OldMask) for a in accounts: - if not 'emailForward' in a: continue + if 'emailForward' not in a: continue Line = "%s: %s" % (a['uid'], a['emailForward']) Line = Sanitize(Line) + "\n" F.write(Line) @@ -621,28 +648,24 @@ def GenForward(accounts, File): Done(File, F, None) def GenCDB(accounts, File, key): - Fdb = None + prefix = ["/usr/bin/eatmydata"] if os.path.exists('/usr/bin/eatmydata') else [] + # nothing else does the fsync stuff, so why do it here? + Fdb = subprocess.Popen(prefix + ["cdbmake", File, "%s.tmp" % File], + preexec_fn=lambda: os.umask(0022), + stdin=subprocess.PIPE) try: - OldMask = os.umask(0022) - # nothing else does the fsync stuff, so why do it here? - prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else '' - Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w") - os.umask(OldMask) - # Write out the email address for each user for a in accounts: - if not key in a: continue + if key not in a: continue value = a[key] user = a['uid'] - Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value)) + Fdb.stdin.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value)) - Fdb.write("\n") - # Oops, something unspeakable happened. - except: - Fdb.close() - raise - if Fdb.close() != None: - raise "cdbmake gave an error" + Fdb.stdin.write("\n") + finally: + Fdb.stdin.close() + if Fdb.wait() != 0: + raise Exception("cdbmake gave an error") def GenDBM(accounts, File, key): Fdb = None @@ -659,7 +682,7 @@ def GenDBM(accounts, File, key): # Write out the email address for each user for a in accounts: - if not key in a: continue + if key not in a: continue value = a[key] user = a['uid'] Fdb[user] = value @@ -704,7 +727,7 @@ def GenPrivate(accounts, File): for a in accounts: if not a.is_active_user(): continue if a.is_guest_account(): continue - if not 'privateSub' in a: continue + if 'privateSub' not in a: continue try: Line = "%s"%(a['privateSub']) Line = Sanitize(Line) + "\n" @@ -746,7 +769,7 @@ def GenMailDisable(accounts, File): F = open(File + ".tmp", "w") for a in accounts: - if not 'mailDisableMessage' in a: continue + if 'mailDisableMessage' not in a: continue Line = "%s: %s"%(a['uid'], a['mailDisableMessage']) Line = Sanitize(Line) + "\n" F.write(Line) @@ -764,7 +787,7 @@ def GenMailBool(accounts, File, key): F = open(File + ".tmp", "w") for a in accounts: - if not key in a: continue + if key not in a: continue if not a[key] == 'TRUE': continue Line = "%s"%(a['uid']) Line = Sanitize(Line) + "\n" @@ -786,7 +809,7 @@ def GenMailList(accounts, File, key): else: validregex = re.compile('^[-\w.]+$') for a in accounts: - if not key in a: continue + if key not in a: continue filtered = filter(lambda z: validregex.match(z), a[key]) if len(filtered) == 0: continue @@ -815,7 +838,7 @@ def GenDNS(accounts, File): # Write out the zone file entry for each user for a in accounts: - if not 'dnsZoneEntry' in a: continue + if 'dnsZoneEntry' not in a: continue if not a.is_active_user() and not isRoleAccount(a): continue if a.is_guest_account(): continue @@ -828,7 +851,7 @@ def GenDNS(accounts, File): F.write(Line) Host = Split[0] + DNSZone - if BSMTPCheck.match(Line) != None: + if BSMTPCheck.match(Line) is not None: F.write("; Has BSMTP\n") # Write some identification information @@ -864,6 +887,7 @@ def is_ipv6_addr(i): return True def ExtractDNSInfo(x): + hostname = GetAttr(x, "hostname") TTLprefix="\t" if 'dnsTTL' in x[1]: @@ -873,38 +897,54 @@ def ExtractDNSInfo(x): if x[1].has_key("ipHostNumber"): for I in x[1]["ipHostNumber"]: if is_ipv6_addr(I): - DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I)) + DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I)) else: - DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I)) + DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I)) Algorithm = None + ssh_hostnames = [ hostname ] + if x[1].has_key("sshfpHostname"): + ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ] + if 'sshRSAHostKey' in x[1]: for I in x[1]["sshRSAHostKey"]: Split = I.split() - if Split[0] == 'ssh-rsa': + key_prefix = Split[0] + key = base64.decodestring(Split[1]) + + # RFC4255 + # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml + if key_prefix == 'ssh-rsa': Algorithm = 1 - if Split[0] == 'ssh-dss': + if key_prefix == 'ssh-dss': Algorithm = 2 - if Algorithm == None: + if key_prefix == 'ssh-ed25519': + Algorithm = 4 + if Algorithm is None: continue - Fingerprint = hashlib.new('sha1', base64.decodestring(Split[1])).hexdigest() - DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint)) + # and more from the registry + sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ] + + fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ] + for h in ssh_hostnames: + for digest_codepoint, fingerprint in fingerprints: + DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint)) if 'architecture' in x[1]: Arch = GetAttr(x, "architecture") Mach = "" if x[1].has_key("machine"): Mach = " " + GetAttr(x, "machine") - DNSInfo.append("%sIN\tHINFO\t\"%s%s\" \"%s\"" % (TTLprefix, Arch, Mach, "Debian GNU/Linux")) + DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian")) if x[1].has_key("mXRecord"): for I in x[1]["mXRecord"]: if I in MX_remap: for e in MX_remap[I]: - DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, e)) + DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e)) else: - DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I)) + DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I)) return DNSInfo @@ -922,40 +962,9 @@ def GenZoneRecords(host_attrs, File): if IsDebianHost.match(GetAttr(x, "hostname")) is None: continue - DNSInfo = ExtractDNSInfo(x) - start = True - for Line in DNSInfo: - if start == True: - Line = "%s.\t%s" % (GetAttr(x, "hostname"), Line) - start = False - else: - Line = "\t\t\t%s" % (Line) - + for Line in ExtractDNSInfo(x): F.write(Line + "\n") - # this would write sshfp lines for services on machines - # but we can't yet, since some are cnames and we'll make - # an invalid zonefile - # - # for i in x[1].get("purpose", []): - # m = PurposeHostField.match(i) - # if m: - # m = m.group(1) - # # we ignore [[*..]] entries - # if m.startswith('*'): - # continue - # if m.startswith('-'): - # m = m[1:] - # if m: - # if not m.endswith(HostDomain): - # continue - # if not m.endswith('.'): - # m = m + "." - # for Line in DNSInfo: - # if isSSHFP.match(Line): - # Line = "%s\t%s" % (m, Line) - # F.write(Line + "\n") - # Oops, something unspeakable happened. except: Die(File, F, None) @@ -970,7 +979,7 @@ def GenBSMTP(accounts, File, HomePrefix): # Write out the zone file entry for each user for a in accounts: - if not 'dnsZoneEntry' in a: continue + if 'dnsZoneEntry' not in a: continue if not a.is_active_user(): continue try: @@ -983,7 +992,7 @@ def GenBSMTP(accounts, File, HomePrefix): Line = " ".join(Split) + "\n" Host = Split[0] + DNSZone - if BSMTPCheck.match(Line) != None: + if BSMTPCheck.match(Line) is not None: F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host, a['uid'], HomePrefix, a['uid'], Host)) @@ -1078,7 +1087,7 @@ def GenHosts(host_attrs, File): if IsDebianHost.match(GetAttr(x, "hostname")) is None: continue - if not 'ipHostNumber' in x[1]: + if 'ipHostNumber' not in x[1]: continue addrs = x[1]["ipHostNumber"] @@ -1120,7 +1129,7 @@ def get_accounts(ldap_conn): "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ "mailContentInspectionAction", "webPassword", "rtcPassword",\ - "bATVToken"]) + "bATVToken", "totpSeed", "mailDefaultOptions"]) if passwd_attrs is None: raise UDEmptyList, "No Users" @@ -1133,9 +1142,10 @@ def get_hosts(ldap_conn): # Fetch all the hosts HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\ ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\ - "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"]) + "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture", + "sshfpHostname"]) - if HostAttrs == None: + if HostAttrs is None: raise UDEmptyList, "No Hosts" HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower())) @@ -1191,7 +1201,6 @@ def generate_all(global_dir, ldap_conn): accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts") accounts = filter(lambda x: not IsRetired(x), accounts) - #accounts_DDs = filter(lambda x: IsGidDebian(x), accounts) CheckForward(accounts) @@ -1200,6 +1209,8 @@ def generate_all(global_dir, ldap_conn): GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward') GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction') + GenCDB(accounts, global_dir + "default-mail-options.cdb", 'mailDefaultOptions') + GenDBM(accounts, global_dir + "default-mail-options.db", 'mailDefaultOptions') GenPrivate(accounts, global_dir + "debian-private") GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock') GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting") @@ -1209,19 +1220,19 @@ def generate_all(global_dir, ldap_conn): GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist") GenWebPassword(accounts, global_dir + "web-passwords") GenRtcPassword(accounts, global_dir + "rtc-passwords") + GenTOTPSeed(accounts, global_dir + "users.oath") GenKeyrings(global_dir) # Compatibility. GenForward(accounts, global_dir + "forward-alias") GenAllUsers(accounts, global_dir + 'all-accounts.json') - accounts = filter(lambda a: not a in accounts_disabled, accounts) + accounts = filter(lambda a: a not in accounts_disabled, accounts) ssh_userkeys = GenSSHShadow(global_dir, accounts) GenMarkers(accounts, global_dir + "markers") GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts") GenHosts(host_attrs, global_dir + "debianhosts") - GenSSHGitolite(accounts, host_attrs, global_dir + "ssh-gitolite") GenDNS(accounts, global_dir + "dns-zone") GenZoneRecords(host_attrs, global_dir + "dns-sshfp") @@ -1229,7 +1240,7 @@ def generate_all(global_dir, ldap_conn): setup_group_maps(ldap_conn) for host in host_attrs: - if not "hostname" in host[1]: + if "hostname" not in host[1]: continue generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys) @@ -1275,11 +1286,11 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): # the relevant ssh keys GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host) - if not 'NOPASSWD' in ExtraList: + if 'NOPASSWD' not in ExtraList: GenShadow(accounts, OutDir + "shadow") # Link in global things - if not 'NOMARKERS' in ExtraList: + if 'NOMARKERS' not in ExtraList: DoLink(global_dir, OutDir, "markers") DoLink(global_dir, OutDir, "mail-forward.cdb") DoLink(global_dir, OutDir, "mail-forward.db") @@ -1292,12 +1303,12 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): DoLink(global_dir, OutDir, "mail-rhsbl") DoLink(global_dir, OutDir, "mail-whitelist") DoLink(global_dir, OutDir, "all-accounts.json") + DoLink(global_dir, OutDir, "default-mail-options.cdb") + DoLink(global_dir, OutDir, "default-mail-options.db") GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward') GenDBM(accounts, OutDir + "user-forward.db", 'emailForward') GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken') GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken') - GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions') - GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions') # Compatibility. DoLink(global_dir, OutDir, "forward-alias") @@ -1316,15 +1327,15 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): DoLink(global_dir, OutDir, "debian-private") if 'GITOLITE' in ExtraList: - DoLink(global_dir, OutDir, "ssh-gitolite") + GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host) if 'exportOptions' in host[1]: for entry in host[1]['exportOptions']: v = entry.split('=',1) if v[0] != 'GITOLITE' or len(v) != 2: continue options = v[1].split(',') - group = options.pop(0); + group = options.pop(0) gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts) - if not 'nohosts' in options: + if 'nohosts' not in options: gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts) else: gitolite_hosts = [] @@ -1340,6 +1351,9 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): if 'RTC-PASSWORDS' in ExtraList: DoLink(global_dir, OutDir, "rtc-passwords") + if 'TOTP' in ExtraList: + DoLink(global_dir, OutDir, "users.oath") + if 'KEYRING' in ExtraList: for k in Keyrings: bn = os.path.basename(k) @@ -1476,7 +1490,8 @@ def ud_generate(): if need_update or options.force: msg = 'Update forced' if options.force else 'Update needed' generate_all(generate_dir, l) - mq_notify(options, msg) + if use_mq: + mq_notify(options, msg) last_run = int(time.time()) fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run)) fd.close()