X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=ud-generate;h=858d736cec157afb188bc15539527e6b9a7fd3d4;hb=c66b8117fef6f45e48c16490874dbbe34ec4e6e2;hp=d78dcaabb65189a4ca7b1efa583f7b6e87335dc9;hpb=8b8ed35607427e0242b70b3623bc7a6814ac78b6;p=mirror%2Fuserdir-ldap.git diff --git a/ud-generate b/ud-generate index d78dcaa..858d736 100755 --- a/ud-generate +++ b/ud-generate @@ -28,11 +28,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp -import lockfile +import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl from userdir_ldap import * from userdir_exceptions import * import UDLdap +from xml.etree.ElementTree import Element, SubElement, Comment +from xml.etree import ElementTree +from xml.dom import minidom try: from cStringIO import StringIO except ImportError: @@ -60,7 +62,7 @@ SubGroupMap = None UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' -EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)?$") +EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$") BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL) PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]") IsV6Addr = re.compile("^[a-fA-F0-9:]+$") @@ -70,6 +72,12 @@ DNSZone = ".debian.net" Keyrings = ConfModule.sync_keyrings.split(":") GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None) +def prettify(elem): + """Return a pretty-printed XML string for the Element. + """ + rough_string = ElementTree.tostring(elem, 'utf-8') + reparsed = minidom.parseString(rough_string) + return reparsed.toprettyxml(indent=" ") def safe_makedirs(dir): try: @@ -89,25 +97,23 @@ def safe_rmtree(dir): else: raise e -def get_lock(fn, wait=5*60, max_age=3600*6): - try: - stat = os.stat(fn + '.lock') - if stat.st_mtime < time.time() - max_age: - sys.stderr.write("Removing stale lock %s"%(fn + '.lock')) - os.unlink(fn + '.lock') - except OSError, error: - if error.errno == errno.ENOENT: - pass - else: - raise - - lock = lockfile.FileLock(fn) - try: - lock.acquire(timeout=wait) - except lockfile.LockTimeout: - return None +def get_lock(fn, wait=5*60): + f = open(fn, "w") + sl = 0.1 + ends = time.time() + wait - return lock + while True: + success = False + try: + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return f + except IOError: + pass + if time.time() >= ends: + return None + sl = min(sl*2, 10, ends - time.time()) + time.sleep(sl) + return None def Sanitize(Str): @@ -393,6 +399,44 @@ def GenWebPassword(accounts, File): Die(File, None, F) raise +# Generate the voipPassword list +def GenVoipPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + root = Element('include') + + for a in accounts: + if not 'voipPassword' in a: continue + if not a.pw_active(): continue + + Pass = str(a['voipPassword']) + user = Element('user') + user.attrib['id'] = "%s" % (a['uid']) + root.append(user) + params = Element('params') + user.append(params) + param = Element('param') + params.append(param) + param.attrib['name'] = "a1-hash" + param.attrib['value'] = "%s" % (Pass) + variables = Element('variables') + user.append(variables) + variable = Element('variable') + variable.attrib['name'] = "toll_allow" + variable.attrib['value'] = "domestic,international,local" + variables.append(variable) + + F.write("%s" % (prettify(root))) + + + except: + Die(File, None, F) + raise + def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host): OldMask = os.umask(0077) tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz') @@ -570,7 +614,9 @@ def GenCDB(accounts, File, key): Fdb = None try: OldMask = os.umask(0022) - Fdb = os.popen("cdbmake %s %s.tmp"%(File, File), "w") + # nothing else does the fsync stuff, so why do it here? + prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else '' + Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w") os.umask(OldMask) # Write out the email address for each user @@ -619,6 +665,7 @@ def GenPrivate(accounts, File): # Write out the position for each user for a in accounts: if not a.is_active_user(): continue + if a.is_guest_account(): continue if not 'privateSub' in a: continue try: Line = "%s"%(a['privateSub']) @@ -732,6 +779,7 @@ def GenDNS(accounts, File): for a in accounts: if not 'dnsZoneEntry' in a: continue if not a.is_active_user() and not isRoleAccount(a): continue + if a.is_guest_account(): continue try: F.write("; %s\n"%(a.email_address())) @@ -913,7 +961,7 @@ def HostToIP(Host, mapped=True): return IPAdresses # Generate the ssh known hosts file -def GenSSHKnown(host_attrs, File, mode=None): +def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None): F = None try: OldMask = os.umask(0022) @@ -953,7 +1001,9 @@ def GenSSHKnown(host_attrs, File, mode=None): hosts = HostToIP(x) if 'sshdistAuthKeysHost' in x[1]: hosts += x[1]['sshdistAuthKeysHost'] - Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (Host, ",".join(hosts), I) + clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host) + clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand) + Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I) else: Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I) Line = Sanitize(Line) + "\n" @@ -1020,7 +1070,7 @@ def get_accounts(ldap_conn): "keyFingerPrint", "privateSub", "mailDisableMessage",\ "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ - "mailContentInspectionAction", "webPassword"]) + "mailContentInspectionAction", "webPassword", "voipPassword"]) if passwd_attrs is None: raise UDEmptyList, "No Users" @@ -1099,13 +1149,14 @@ def generate_all(global_dir, ldap_conn): GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward') GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') GenPrivate(accounts, global_dir + "debian-private") - GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys') + GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock') GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting") GenMailBool(accounts, global_dir + "mail-callout", "mailCallout") GenMailList(accounts, global_dir + "mail-rbl", "mailRBL") GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL") GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist") GenWebPassword(accounts, global_dir + "web-passwords") + GenVoipPassword(accounts, global_dir + "voip-passwords") GenKeyrings(global_dir) # Compatibility. @@ -1213,6 +1264,9 @@ def generate_host(host, global_dir, accounts, ssh_userkeys): if 'WEB-PASSWORDS' in ExtraList: DoLink(global_dir, OutDir, "web-passwords") + if 'VOIP-PASSWORDS' in ExtraList: + DoLink(global_dir, OutDir, "voip-passwords") + if 'KEYRING' in ExtraList: for k in Keyrings: bn = os.path.basename(k) @@ -1250,15 +1304,26 @@ def getLastLDAPChangeTime(l): return last +def getLastKeyringChangeTime(): + krmod = 0 + for k in Keyrings: + mt = os.path.getmtime(k) + if mt > krmod: + krmod = mt + + return krmod + def getLastBuildTime(gdir): - cache_last_mod = 0 + cache_last_ldap_mod = 0 + cache_last_unix_mod = 0 try: fd = open(os.path.join(gdir, "last_update.trace"), "r") cache_last_mod=fd.read().split() try: - cache_last_mod = cache_last_mod[0] - except IndexError: + cache_last_ldap_mod = cache_last_mod[0] + cache_last_unix_mod = int(cache_last_mod[1]) + except IndexError, ValueError: pass fd.close() except IOError, e: @@ -1267,8 +1332,7 @@ def getLastBuildTime(gdir): else: raise e - return cache_last_mod - + return (cache_last_ldap_mod, cache_last_unix_mod) def ud_generate(): parser = optparse.OptionParser() @@ -1282,44 +1346,51 @@ def ud_generate(): parser.print_help() sys.exit(1) - - l = make_ldap_conn() - if options.generatedir is not None: generate_dir = os.environ['UD_GENERATEDIR'] elif 'UD_GENERATEDIR' in os.environ: generate_dir = os.environ['UD_GENERATEDIR'] + else: + generate_dir = GenerateDir + + + lockf = os.path.join(generate_dir, 'ud-generate.lock') + lock = get_lock( lockf ) + if lock is None: + sys.stderr.write("Could not acquire lock %s.\n"%(lockf)) + sys.exit(1) + + l = make_ldap_conn() + time_started = int(time.time()) ldap_last_mod = getLastLDAPChangeTime(l) - cache_last_mod = getLastBuildTime(generate_dir) - need_update = ldap_last_mod > cache_last_mod + unix_last_mod = getLastKeyringChangeTime() + cache_last_ldap_mod, cache_last_unix_mod = getLastBuildTime(generate_dir) + + need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) if not options.force and not need_update: fd = open(os.path.join(generate_dir, "last_update.trace"), "w") - fd.write("%s\n%s\n" % (ldap_last_mod, int(time.time()))) + fd.write("%s\n%s\n" % (ldap_last_mod, time_started)) fd.close() sys.exit(0) - lock = None - try: - lockf = os.path.join(generate_dir, 'ud-generate.lock') - lock = get_lock( lockf ) - if lock is None: - sys.stderr.write("Could not acquire lock %s.\n"%(lockf)) - sys.exit(1) - - tracefd = open(os.path.join(generate_dir, "last_update.trace"), "w") - generate_all(generate_dir, l) - tracefd.write("%s\n%s\n" % (ldap_last_mod, int(time.time()))) - tracefd.close() + tracefd = open(os.path.join(generate_dir, "last_update.trace"), "w") + generate_all(generate_dir, l) + tracefd.write("%s\n%s\n" % (ldap_last_mod, time_started)) + tracefd.close() - finally: - if lock is not None: - lock.release() if __name__ == "__main__": - ud_generate() - + if 'UD_PROFILE' in os.environ: + import cProfile + import pstats + cProfile.run('ud_generate()', "udg_prof") + p = pstats.Stats('udg_prof') + ##p.sort_stats('time').print_stats() + p.sort_stats('cumulative').print_stats() + else: + ud_generate() # vim:set et: # vim:set ts=3: