X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=ud-generate;h=cfd3f2a064c6d08ea3a7bc009b9a218c10824af7;hb=36f0c76cb2b20e2f1df5b4dce102431a5ae72d0f;hp=8bc04328d4a8d625e440e5d888b7b197c06c9799;hpb=66fc64eaf3ca0bde24b7c9656cbf6b0029ab5154;p=mirror%2Fuserdir-ldap.git diff --git a/ud-generate b/ud-generate index 8bc0432..cfd3f2a 100755 --- a/ud-generate +++ b/ud-generate @@ -28,10 +28,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl +import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm from userdir_ldap import * from userdir_exceptions import * import UDLdap +from xml.etree.ElementTree import Element, SubElement, Comment +from xml.etree import ElementTree +from xml.dom import minidom try: from cStringIO import StringIO except ImportError: @@ -58,17 +61,25 @@ SubGroupMap = None UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' +MAX_UD_AGE = 3600*24 -EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)?$") +EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$") BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL) PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]") -IsV6Addr = re.compile("^[a-fA-F0-9:]+$") IsDebianHost = re.compile(ConfModule.dns_hostmatch) isSSHFP = re.compile("^\s*IN\s+SSHFP") DNSZone = ".debian.net" Keyrings = ConfModule.sync_keyrings.split(":") GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None) +GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", ".")) +MX_remap = json.loads(ConfModule.MX_remap) +def prettify(elem): + """Return a pretty-printed XML string for the Element. + """ + rough_string = ElementTree.tostring(elem, 'utf-8') + reparsed = minidom.parseString(rough_string) + return reparsed.toprettyxml(indent=" ") def safe_makedirs(dir): try: @@ -163,7 +174,7 @@ def IsInGroup(account, allowed, current_host): supgroups=[] addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host) for g in supgroups: - if allowed.has_key(g): + if g in allowed: return True return False @@ -326,7 +337,7 @@ def GenShadowSudo(accounts, File, untrusted, current_host): Done(File, F, None) # Generate the sudo passwd file -def GenSSHGitolite(accounts, File): +def GenSSHGitolite(accounts, hosts, File): F = None try: OldMask = os.umask(0022) @@ -347,6 +358,15 @@ def GenSSHGitolite(accounts, File): line = Sanitize(line) + "\n" F.write(line) + for dn, attrs in hosts: + if not 'sshRSAHostKey' in attrs: continue + hostname = "host-" + attrs['hostname'][0] + prefix = GitoliteSSHRestrictions.replace('@@USER@@', hostname) + for I in attrs["sshRSAHostKey"]: + line = "%s %s"%(prefix, I) + line = Sanitize(line) + "\n" + F.write(line) + # Oops, something unspeakable happened. except: Die(File, F, None) @@ -390,6 +410,26 @@ def GenWebPassword(accounts, File): Die(File, None, F) raise +# Generate the voipPassword list +def GenVoipPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + for a in accounts: + if not 'voipPassword' in a: continue + if not a.pw_active(): continue + + Line = "%s@debian.org:%s:sip.debian.org:AUTHORIZED" % (a['uid'], str(a['voipPassword'])) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + + except: + Die(File, None, F) + raise + def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host): OldMask = os.umask(0077) tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz') @@ -587,6 +627,34 @@ def GenCDB(accounts, File, key): if Fdb.close() != None: raise "cdbmake gave an error" +def GenDBM(accounts, File, key): + Fdb = None + OldMask = os.umask(0022) + fn = os.path.join(File).encode('ascii', 'ignore') + try: + posix.remove(fn) + except: + pass + + try: + Fdb = dbm.open(fn, "c") + os.umask(OldMask) + + # Write out the email address for each user + for a in accounts: + if not key in a: continue + value = a[key] + user = a['uid'] + Fdb[user] = value + + Fdb.close() + except: + # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db + os.remove(File + ".db") + raise + # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db + os.rename (File + ".db", File) + # Generate the anon XEarth marker file def GenMarkers(accounts, File): F = None @@ -618,6 +686,7 @@ def GenPrivate(accounts, File): # Write out the position for each user for a in accounts: if not a.is_active_user(): continue + if a.is_guest_account(): continue if not 'privateSub' in a: continue try: Line = "%s"%(a['privateSub']) @@ -731,6 +800,7 @@ def GenDNS(accounts, File): for a in accounts: if not 'dnsZoneEntry' in a: continue if not a.is_active_user() and not isRoleAccount(a): continue + if a.is_guest_account(): continue try: F.write("; %s\n"%(a.email_address())) @@ -769,6 +839,13 @@ def GenDNS(accounts, File): raise Done(File, F, None) +def is_ipv6_addr(i): + try: + socket.inet_pton(socket.AF_INET6, i) + except socket.error: + return False + return True + def ExtractDNSInfo(x): TTLprefix="\t" @@ -778,7 +855,7 @@ def ExtractDNSInfo(x): DNSInfo = [] if x[1].has_key("ipHostNumber"): for I in x[1]["ipHostNumber"]: - if IsV6Addr.match(I) != None: + if is_ipv6_addr(I): DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I)) else: DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I)) @@ -806,7 +883,11 @@ def ExtractDNSInfo(x): if x[1].has_key("mXRecord"): for I in x[1]["mXRecord"]: - DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I)) + if I in MX_remap: + for e in MX_remap[I]: + DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, e)) + else: + DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I)) return DNSInfo @@ -906,13 +987,13 @@ def HostToIP(Host, mapped=True): if Host[1].has_key("ipHostNumber"): for addr in Host[1]["ipHostNumber"]: IPAdresses.append(addr) - if IsV6Addr.match(addr) is None and mapped == "True": + if not is_ipv6_addr(addr) and mapped == "True": IPAdresses.append("::ffff:"+addr) return IPAdresses # Generate the ssh known hosts file -def GenSSHKnown(host_attrs, File, mode=None): +def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None): F = None try: OldMask = os.umask(0022) @@ -952,7 +1033,9 @@ def GenSSHKnown(host_attrs, File, mode=None): hosts = HostToIP(x) if 'sshdistAuthKeysHost' in x[1]: hosts += x[1]['sshdistAuthKeysHost'] - Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (Host, ",".join(hosts), I) + clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host) + clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand) + Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I) else: Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I) Line = Sanitize(Line) + "\n" @@ -1019,7 +1102,7 @@ def get_accounts(ldap_conn): "keyFingerPrint", "privateSub", "mailDisableMessage",\ "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ - "mailContentInspectionAction", "webPassword"]) + "mailContentInspectionAction", "webPassword", "voipPassword"]) if passwd_attrs is None: raise UDEmptyList, "No Users" @@ -1096,15 +1179,18 @@ def generate_all(global_dir, ldap_conn): GenMailDisable(accounts, global_dir + "mail-disable") GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward') + GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward') GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') + GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction') GenPrivate(accounts, global_dir + "debian-private") - GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys') + GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock') GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting") GenMailBool(accounts, global_dir + "mail-callout", "mailCallout") GenMailList(accounts, global_dir + "mail-rbl", "mailRBL") GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL") GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist") GenWebPassword(accounts, global_dir + "web-passwords") + GenVoipPassword(accounts, global_dir + "voip-passwords") GenKeyrings(global_dir) # Compatibility. @@ -1117,7 +1203,7 @@ def generate_all(global_dir, ldap_conn): GenMarkers(accounts, global_dir + "markers") GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts") GenHosts(host_attrs, global_dir + "debianhosts") - GenSSHGitolite(accounts, global_dir + "ssh-gitolite") + GenSSHGitolite(accounts, host_attrs, global_dir + "ssh-gitolite") GenDNS(accounts, global_dir + "dns-zone") GenZoneRecords(host_attrs, global_dir + "dns-sshfp") @@ -1127,9 +1213,9 @@ def generate_all(global_dir, ldap_conn): for host in host_attrs: if not "hostname" in host[1]: continue - generate_host(host, global_dir, accounts, ssh_userkeys) + generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys) -def generate_host(host, global_dir, accounts, ssh_userkeys): +def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys): current_host = host[1]['hostname'][0] OutDir = global_dir + current_host + '/' if not os.path.isdir(OutDir): @@ -1152,7 +1238,7 @@ def generate_host(host, global_dir, accounts, ssh_userkeys): ExtraList[extra.upper()] = True if GroupList != {}: - accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), accounts) + accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts) DoLink(global_dir, OutDir, "debianhosts") DoLink(global_dir, OutDir, "ssh_known_hosts") @@ -1178,7 +1264,9 @@ def generate_host(host, global_dir, accounts, ssh_userkeys): if not 'NOMARKERS' in ExtraList: DoLink(global_dir, OutDir, "markers") DoLink(global_dir, OutDir, "mail-forward.cdb") + DoLink(global_dir, OutDir, "mail-forward.db") DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb") + DoLink(global_dir, OutDir, "mail-contentinspectionaction.db") DoLink(global_dir, OutDir, "mail-disable") DoLink(global_dir, OutDir, "mail-greylist") DoLink(global_dir, OutDir, "mail-callout") @@ -1187,8 +1275,11 @@ def generate_host(host, global_dir, accounts, ssh_userkeys): DoLink(global_dir, OutDir, "mail-whitelist") DoLink(global_dir, OutDir, "all-accounts.json") GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward') + GenDBM(accounts, OutDir + "user-forward.db", 'emailForward') GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken') + GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken') GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions') + GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions') # Compatibility. DoLink(global_dir, OutDir, "forward-alias") @@ -1208,10 +1299,20 @@ def generate_host(host, global_dir, accounts, ssh_userkeys): if 'GITOLITE' in ExtraList: DoLink(global_dir, OutDir, "ssh-gitolite") + if 'exportOptions' in host[1]: + for entry in host[1]['exportOptions']: + v = entry.split('=',1) + if v[0] != 'GITOLITE' or len(v) != 2: continue + gitolite_accounts = filter(lambda x: IsInGroup(x, [v[1]], current_host), all_accounts) + gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts) + GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(v[1],)) if 'WEB-PASSWORDS' in ExtraList: DoLink(global_dir, OutDir, "web-passwords") + if 'VOIP-PASSWORDS' in ExtraList: + DoLink(global_dir, OutDir, "voip-passwords") + if 'KEYRING' in ExtraList: for k in Keyrings: bn = os.path.basename(k) @@ -1249,15 +1350,28 @@ def getLastLDAPChangeTime(l): return last +def getLastKeyringChangeTime(): + krmod = 0 + for k in Keyrings: + mt = os.path.getmtime(k) + if mt > krmod: + krmod = mt + + return int(krmod) + def getLastBuildTime(gdir): - cache_last_mod = 0 + cache_last_ldap_mod = 0 + cache_last_unix_mod = 0 + cache_last_run = 0 try: fd = open(os.path.join(gdir, "last_update.trace"), "r") cache_last_mod=fd.read().split() try: - cache_last_mod = cache_last_mod[0] - except IndexError: + cache_last_ldap_mod = cache_last_mod[0] + cache_last_unix_mod = int(cache_last_mod[1]) + cache_last_run = int(cache_last_mod[2]) + except IndexError, ValueError: pass fd.close() except IOError, e: @@ -1266,15 +1380,14 @@ def getLastBuildTime(gdir): else: raise e - return cache_last_mod - + return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run) def ud_generate(): parser = optparse.OptionParser() parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR", help="Output directory.") parser.add_option("-f", "--force", dest="force", action="store_true", - help="Force generation, even if not update to LDAP has happened.") + help="Force generation, even if no update to LDAP has happened.") (options, args) = parser.parse_args() if len(args) > 0: @@ -1297,19 +1410,22 @@ def ud_generate(): l = make_ldap_conn() + time_started = int(time.time()) ldap_last_mod = getLastLDAPChangeTime(l) - cache_last_mod = getLastBuildTime(generate_dir) - need_update = ldap_last_mod > cache_last_mod + unix_last_mod = getLastKeyringChangeTime() + cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir) + + need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE) if not options.force and not need_update: fd = open(os.path.join(generate_dir, "last_update.trace"), "w") - fd.write("%s\n%s\n" % (ldap_last_mod, int(time.time()))) + fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run)) fd.close() sys.exit(0) tracefd = open(os.path.join(generate_dir, "last_update.trace"), "w") generate_all(generate_dir, l) - tracefd.write("%s\n%s\n" % (ldap_last_mod, int(time.time()))) + tracefd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, time_started)) tracefd.close()