Update ud-ldapshow and cleanup cruft around the usergroups changes

[mirror/userdir-ldap.git] / ud-generate

diff --git a/ud-generate b/ud-generate
index bbddc2f..7b9ca4c 100755 (executable)
--- a/ud-generate
+++ b/ud-generate
@@ -6,11 +6,13 @@
 #   Copyright (c) 2003-2004  James Troup <troup@debian.org>
 #   Copyright (c) 2004-2005,7  Joey Schulze <joey@infodrom.org>
 #   Copyright (c) 2001-2007  Ryan Murray <rmurray@debian.org>
 #   Copyright (c) 2003-2004  James Troup <troup@debian.org>
 #   Copyright (c) 2004-2005,7  Joey Schulze <joey@infodrom.org>
 #   Copyright (c) 2001-2007  Ryan Murray <rmurray@debian.org>

-#   Copyright (c) 2008 Peter Palfrader <peter@palfrader.org>
+#   Copyright (c) 2008,2009,2010,2011 Peter Palfrader <peter@palfrader.org>

 #   Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
 #   Copyright (c) 2008 Mark Hymers <mhy@debian.org>
 #   Copyright (c) 2008 Luk Claes <luk@debian.org>
 #   Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>
 #   Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
 #   Copyright (c) 2008 Mark Hymers <mhy@debian.org>
 #   Copyright (c) 2008 Luk Claes <luk@debian.org>
 #   Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>

+#   Copyright (c) 2009 Stephen Gran <steve@lobefin.net>
+#   Copyright (c) 2010 Helmut Grohne <helmut@subdivi.de>

 #
 #   This program is free software; you can redistribute it and/or modify
 #   it under the terms of the GNU General Public License as published by
 #
 #   This program is free software; you can redistribute it and/or modify
 #   it under the terms of the GNU General Public License as published by


@@ -26,333 +28,428 @@
 #   along with this program; if not, write to the Free Software
 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 
 #   along with this program; if not, write to the Free Software
 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 

-import string, re, time, ldap, getopt, sys, os, pwd, posix, socket, base64, sha, shutil, errno, tarfile, grp
-from userdir_ldap import *;
+from dsa_mq.connection import Connection
+from dsa_mq.config import Config
+
+import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
+from userdir_ldap import *
+from userdir_exceptions import *
+import UDLdap
+from xml.etree.ElementTree import Element, SubElement, Comment
+from xml.etree import ElementTree
+from xml.dom import minidom
+try:
+   from cStringIO import StringIO
+except ImportError:
+   from StringIO import StringIO
+try:
+   import simplejson as json
+except ImportError:
+   import json
+   if not '__author__' in json.__dict__:
+      sys.stderr.write("Warning: This is probably the wrong json module.  We want python 2.6's json\n")
+      sys.stderr.write("module, or simplejson on pytyon 2.5.  Let's see if/how stuff blows up.\n")
+
+if os.getuid() == 0:
+   sys.stderr.write("You should probably not run ud-generate as root.\n")
+   sys.exit(1)
+
+
+#
+# GLOBAL STATE
+#
+GroupIDMap = None
+SubGroupMap = None

 
 

-global Allowed;
-global CurrentHost;

 
 

-PasswdAttrs = None;
-disabledusers = []
-GroupIDMap = {};
-SubGroupMap = {};
-Allowed = None;
-CurrentHost = "";

 
 UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
 
 UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'

+MAX_UD_AGE = 3600*24

 
 

-EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)?$");
-BSMTPCheck = re.compile(".*mx 0 (gluck)\.debian\.org\..*",re.DOTALL);
-PurposeHostField = re.compile(r"\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
+EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
+BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL)
+PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
+IsDebianHost = re.compile(ConfModule.dns_hostmatch)
+isSSHFP = re.compile("^\s*IN\s+SSHFP")

 DNSZone = ".debian.net"
 Keyrings = ConfModule.sync_keyrings.split(":")
 DNSZone = ".debian.net"
 Keyrings = ConfModule.sync_keyrings.split(":")

+GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
+GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
+GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
+MX_remap = json.loads(ConfModule.MX_remap)
+
+def prettify(elem):
+   """Return a pretty-printed XML string for the Element.
+   """
+   rough_string = ElementTree.tostring(elem, 'utf-8')
+   reparsed = minidom.parseString(rough_string)
+   return reparsed.toprettyxml(indent="  ")

 
 def safe_makedirs(dir):
 
 def safe_makedirs(dir):

-    try:
-        os.makedirs(dir)
-    except OSError, e:
-        if e.errno == errno.EEXIST:
-            pass
-        else:
-            raise e
+   try:
+      os.makedirs(dir)
+   except OSError, e:
+      if e.errno == errno.EEXIST:
+         pass
+      else:
+         raise e

 
 def safe_rmtree(dir):
 
 def safe_rmtree(dir):

-    try:
-        shutil.rmtree(dir)
-    except OSError, e:
-        if e.errno == errno.ENOENT:
-            pass
-        else:
-            raise e
+   try:
+      shutil.rmtree(dir)
+   except OSError, e:
+      if e.errno == errno.ENOENT:
+         pass
+      else:
+         raise e
+
+def get_lock(fn, wait=5*60):
+   f = open(fn, "w")
+   sl = 0.1
+   ends = time.time() + wait
+
+   while True:
+      success = False
+      try:
+         fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+         return f
+      except IOError:
+         pass
+      if time.time() >= ends:
+         return None
+      sl = min(sl*2, 10, ends - time.time())
+      time.sleep(sl)
+   return None
+

 
 def Sanitize(Str):
 
 def Sanitize(Str):

-  return Str.translate(string.maketrans("\n\r\t","$$$"))
+   return Str.translate(string.maketrans("\n\r\t", "$$$"))

 
 

-def DoLink(From,To,File):
-   try: posix.remove(To+File);
-   except: pass;
-   posix.link(From+File,To+File);
+def DoLink(From, To, File):
+   try: 
+      posix.remove(To + File)
+   except: 
+      pass
+   posix.link(From + File, To + File)

 
 

-def IsRetired(DnRecord):
+def IsRetired(account):

    """
    Looks for accountStatus in the LDAP record and tries to
    match it against one of the known retired statuses
    """
 
    """
    Looks for accountStatus in the LDAP record and tries to
    match it against one of the known retired statuses
    """
 

-   status = GetAttr(DnRecord,"accountStatus", None)
-   if status is None:
-      return False
+   status = account['accountStatus']

 
 

-   if status.find("inactive") != -1:
+   line = status.split()
+   status = line[0]
+
+   if status == "inactive":

       return True
 
       return True
 

-   if status.find("memorial") != -1:
+   elif status == "memorial":

       return True
 
       return True
 

-   if status.find("retiring") != -1:
-      line = status.split()
+   elif status == "retiring":

       # We'll give them a few extra days over what we said
       age = 6 * 31 * 24 * 60 * 60
       # We'll give them a few extra days over what we said
       age = 6 * 31 * 24 * 60 * 60

-      if (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d")) > (age):
-            return True
+      try:
+         return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age
+      except IndexError:
+         return False
+      except ValueError:
+         return False

 
    return False
 
 # See if this user is in the group list
 
    return False
 
 # See if this user is in the group list

-def IsInGroup(DnRecord):
-  if Allowed == None:
-     return 1;
-
+def IsInGroup(account, allowed, current_host):

   # See if the primary group is in the list
   # See if the primary group is in the list

-  if Allowed.has_key(GetAttr(DnRecord,"gidNumber")) != 0:
-     return 1;
+  if str(account['gidNumber']) in allowed: return True

 
   # Check the host based ACL
 
   # Check the host based ACL

-  if DnRecord[1].has_key("allowedHost") != 0:
-     for I in DnRecord[1]["allowedHost"]:
-        if CurrentHost == I:
-           return 1;
+  if account.is_allowed_by_hostacl(current_host): return True

 
   # See if there are supplementary groups
 
   # See if there are supplementary groups

-  if DnRecord[1].has_key("supplementaryGid") == 0:
-     return 0;
+  if not 'supplementaryGid' in account: return False

 
   supgroups=[]
 
   supgroups=[]

-  addGroups(supgroups, DnRecord[1]["supplementaryGid"], GetAttr(DnRecord,"uid"))
+  addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)

   for g in supgroups:
   for g in supgroups:

-     if Allowed.has_key(g):
-        return 1;
-  return 0;
+     if g in allowed:
+        return True
+  return False

 
 

-def Die(File,F,Fdb):
+def Die(File, F, Fdb):
+   if F != None:
+      F.close()
+   if Fdb != None:
+      Fdb.close()
+   try: 
+      os.remove(File + ".tmp")
+   except:
+      pass
+   try: 
+      os.remove(File + ".tdb.tmp")
+   except: 
+      pass
+
+def Done(File, F, Fdb):

    if F != None:
    if F != None:

-      F.close();
+      F.close()
+      os.rename(File + ".tmp", File)

    if Fdb != None:
    if Fdb != None:

-      Fdb.close();
-   try: os.remove(File + ".tmp");
-   except: pass;
-   try: os.remove(File + ".tdb.tmp");
-   except: pass;
-
-def Done(File,F,Fdb):
-  if F != None:
-    F.close();
-    os.rename(File + ".tmp",File);
-  if Fdb != None:
-    Fdb.close();
-    os.rename(File + ".tdb.tmp",File+".tdb");
+      Fdb.close()
+      os.rename(File + ".tdb.tmp", File + ".tdb")

 
 # Generate the password list
 
 # Generate the password list

-def GenPasswd(l,File,HomePrefix,PwdMarker):
-  F = None;
-  try:
-   F = open(File + ".tdb.tmp","w");
-
-   userlist = {}
-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   I = 0;
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
-
-      if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0:
-         continue;
-
-      # Do not let people try to buffer overflow some busted passwd parser.
-      if len(GetAttr(x,"gecos")) > 100 or len(GetAttr(x,"loginShell")) > 50:
-         continue;
-
-      userlist[GetAttr(x, "uid")] = int(GetAttr(x, "gidNumber"))
-      Line = "%s:%s:%s:%s:%s:%s%s:%s" % (GetAttr(x,"uid"),\
-              PwdMarker,\
-              GetAttr(x,"uidNumber"),GetAttr(x,"gidNumber"),\
-              GetAttr(x,"gecos"),HomePrefix,GetAttr(x,"uid"),\
-              GetAttr(x,"loginShell"));
-
-      Line = Sanitize(Line) + "\n";
-      F.write("0%u %s" % (I,Line));
-      F.write(".%s %s" % (GetAttr(x,"uid"),Line));
-      F.write("=%s %s" % (GetAttr(x,"uidNumber"),Line));
-      I = I + 1;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,None,F);
-   raise;
-  Done(File,None,F);
-
-  # Return the list of users so we know which keys to export
-  return userlist
+def GenPasswd(accounts, File, HomePrefix, PwdMarker):
+   F = None
+   try:
+      F = open(File + ".tdb.tmp", "w")
+
+      userlist = {}
+      i = 0
+      for a in accounts:
+         # Do not let people try to buffer overflow some busted passwd parser.
+         if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue
+
+         userlist[a['uid']] = a['gidNumber']
+         line = "%s:%s:%d:%d:%s:%s%s:%s" % (
+                 a['uid'],
+                 PwdMarker,
+                 a['uidNumber'],
+                 a['gidNumber'],
+                 a['gecos'],
+                 HomePrefix, a['uid'],
+                 a['loginShell'])
+         line = Sanitize(line) + "\n"
+         F.write("0%u %s" % (i, line))
+         F.write(".%s %s" % (a['uid'], line))
+         F.write("=%d %s" % (a['uidNumber'], line))
+         i = i + 1
+
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, None, F)
+      raise
+   Done(File, None, F)
+
+   # Return the list of users so we know which keys to export
+   return userlist
+
+def GenAllUsers(accounts, file):
+   f = None
+   try:
+      OldMask = os.umask(0022)
+      f = open(file + ".tmp", "w", 0644)
+      os.umask(OldMask)
+
+      all = []
+      for a in accounts:
+         all.append( { 'uid': a['uid'],
+                       'uidNumber': a['uidNumber'],
+                       'active': a.pw_active() and a.shadow_active() } )
+      json.dump(all, f)
+
+   # Oops, something unspeakable happened.
+   except:
+      Die(file, f, None)
+      raise
+   Done(file, f, None)

 
 # Generate the shadow list
 
 # Generate the shadow list

-def GenShadow(l,File):
-  F = None;
-  try:
-   OldMask = os.umask(0077);
-   F = open(File + ".tdb.tmp","w",0600);
-   os.umask(OldMask);
-
-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   I = 0;
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
-
-      if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0:
-         continue;
-
-      Pass = GetAttr(x,"userPassword");
-      if Pass[0:7] != "{crypt}" or len(Pass) > 50:
-         Pass = '*';
-      else:
-         Pass = Pass[7:];
-
-      # If the account is locked, mark it as such in shadow
-      # See Debian Bug #308229 for why we set it to 1 instead of 0
-      if (GetAttr(x,"userPassword").find("*LK*") != -1) \
-          or GetAttr(x,"userPassword").startswith("!"):
-         ShadowExpire = '1'
-      else:
-         ShadowExpire = GetAttr(x,"shadowExpire")
-
-      Line = "%s:%s:%s:%s:%s:%s:%s:%s:" % (GetAttr(x,"uid"),\
-              Pass,GetAttr(x,"shadowLastChange"),\
-              GetAttr(x,"shadowMin"),GetAttr(x,"shadowMax"),\
-              GetAttr(x,"shadowWarning"),GetAttr(x,"shadowInactive"),\
-              ShadowExpire);
-      Line = Sanitize(Line) + "\n";
-      F.write("0%u %s" % (I,Line));
-      F.write(".%s %s" % (GetAttr(x,"uid"),Line));
-      I = I + 1;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,None,F);
-   raise;
-  Done(File,None,F);
+def GenShadow(accounts, File):
+   F = None
+   try:
+      OldMask = os.umask(0077)
+      F = open(File + ".tdb.tmp", "w", 0600)
+      os.umask(OldMask)
+
+      i = 0
+      for a in accounts:
+         # If the account is locked, mark it as such in shadow
+         # See Debian Bug #308229 for why we set it to 1 instead of 0
+         if not a.pw_active():     ShadowExpire = '1'
+         elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire'])
+         else:                     ShadowExpire = ''
+
+         values = []
+         values.append(a['uid'])
+         values.append(a.get_password())
+         for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive':
+            if key in a: values.append(a[key])
+            else:        values.append('')
+         values.append(ShadowExpire)
+         line = ':'.join(values)+':'
+         line = Sanitize(line) + "\n"
+         F.write("0%u %s" % (i, line))
+         F.write(".%s %s" % (a['uid'], line))
+         i = i + 1
+
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, None, F)
+      raise
+   Done(File, None, F)

 
 # Generate the sudo passwd file
 
 # Generate the sudo passwd file

-def GenShadowSudo(l,File, untrusted):
-  F = None;
-  try:
-   OldMask = os.umask(0077);
-   F = open(File + ".tmp","w",0600);
-   os.umask(OldMask);
-
-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+def GenShadowSudo(accounts, File, untrusted, current_host):
+   F = None
+   try:
+      OldMask = os.umask(0077)
+      F = open(File + ".tmp", "w", 0600)
+      os.umask(OldMask)
+
+      for a in accounts:
+         Pass = '*'
+         if 'sudoPassword' in a:
+            for entry in a['sudoPassword']:
+               Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
+               if Match == None:
+                  continue
+               uuid = Match.group(1)
+               status = Match.group(2)
+               hosts = Match.group(3)
+               cryptedpass = Match.group(4)
+     
+               if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
+                  continue
+               for_all = hosts == "*"
+               for_this_host = current_host in hosts.split(',')
+               if not (for_all or for_this_host):
+                  continue
+               # ignore * passwords for untrusted hosts, but copy host specific passwords
+               if for_all and untrusted:
+                  continue
+               Pass = cryptedpass
+               if for_this_host: # this makes sure we take a per-host entry over the for-all entry
+                  break
+            if len(Pass) > 50:
+               Pass = '*'
+     
+         Line = "%s:%s" % (a['uid'], Pass)
+         Line = Sanitize(Line) + "\n"
+         F.write("%s" % (Line))
+  
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-      Pass = '*'
-      if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0:
-         continue;
-
-      if x[1].has_key('sudoPassword'):
-         for entry in x[1]['sudoPassword']:
-            Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry)
-            if Match == None:
-               continue
-            uuid = Match.group(1)
-            status = Match.group(2)
-            hosts = Match.group(3)
-            cryptedpass = Match.group(4)
-
-            if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', x[1]['uid'][0], uuid, hosts, cryptedpass):
-               continue
-            for_all = hosts == "*"
-            for_this_host = CurrentHost in hosts.split(',')
-            if not (for_all or for_this_host):
-               continue
-            # ignore * passwords for untrusted hosts, but copy host specific passwords
-            if for_all and untrusted:
-               continue
-            Pass = cryptedpass
-            if for_this_host: # this makes sure we take a per-host entry over the for-all entry
-              break
-         if len(Pass) > 50:
-            Pass = '*'
-
-      Line = "%s:%s" % (GetAttr(x,"uid"), Pass)
-      Line = Sanitize(Line) + "\n";
-      F.write("%s" % (Line));
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
+# Generate the sudo passwd file
+def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
+   F = None
+   if sshcommand is None:
+      sshcommand = GitoliteSSHCommand
+   try:
+      OldMask = os.umask(0022)
+      F = open(File + ".tmp", "w", 0600)
+      os.umask(OldMask)
+
+      if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
+         for a in accounts:
+            if not 'sshRSAAuthKey' in a: continue
+
+            User = a['uid']
+            prefix = GitoliteSSHRestrictions
+            prefix = prefix.replace('@@COMMAND@@', sshcommand)
+            prefix = prefix.replace('@@USER@@', User)
+            for I in a["sshRSAAuthKey"]:
+               if I.startswith("allowed_hosts=") and ' ' in line:
+                  if current_host is None:
+                     continue
+                  machines, I = I.split('=', 1)[1].split(' ', 1)
+                  if current_host not in machines.split(','):
+                     continue # skip this key
+
+               if I.startswith('ssh-'):
+                  line = "%s %s"%(prefix, I)
+               else:
+                  continue # do not allow keys with other restrictions that might conflict
+               line = Sanitize(line) + "\n"
+               F.write(line)
+
+         for dn, attrs in hosts:
+            if not 'sshRSAHostKey' in attrs: continue
+            hostname = "host-" + attrs['hostname'][0]
+            prefix = GitoliteSSHRestrictions
+            prefix = prefix.replace('@@COMMAND@@', sshcommand)
+            prefix = prefix.replace('@@USER@@', hostname)
+            for I in attrs["sshRSAHostKey"]:
+               line = "%s %s"%(prefix, I)
+               line = Sanitize(line) + "\n"
+               F.write(line)
+
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 # Generate the shadow list
 
 # Generate the shadow list

-def GenSSHShadow(l):
+def GenSSHShadow(global_dir, accounts):

    # Fetch all the users
    # Fetch all the users

-   singlefile = None
-   userfiles = []
-
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   safe_rmtree(os.path.join(GlobalDir, 'userkeys'))
-   safe_makedirs(os.path.join(GlobalDir, 'userkeys'))
-
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+   userkeys = {}
+
+   for a in accounts:
+      if not 'sshRSAAuthKey' in a: continue
+
+      contents = []
+      for I in a['sshRSAAuthKey']:
+         MultipleLine = "%s" % I
+         MultipleLine = Sanitize(MultipleLine)
+         contents.append(MultipleLine)
+      userkeys[a['uid']] = contents
+   return userkeys
+
+# Generate the webPassword list
+def GenWebPassword(accounts, File):
+   F = None
+   try:
+      OldMask = os.umask(0077)
+      F = open(File, "w", 0600)
+      os.umask(OldMask)

 
 

-      # If the account is locked, do not write it.
-      # This is a partial stop-gap. The ssh also needs to change this
-      # to ignore ~/.ssh/authorized* files.
-      if (GetAttr(x,"userPassword").find("*LK*") != -1) \
-             or GetAttr(x,"userPassword").startswith("!"):
-         continue;
+      for a in accounts:
+         if not 'webPassword' in a: continue
+         if not a.pw_active(): continue

 
 

-      if x[1].has_key("uidNumber") == 0 or \
-         x[1].has_key("sshRSAAuthKey") == 0:
-         continue;
-      User = GetAttr(x,"uid");
-      F = None;
+         Pass = str(a['webPassword'])
+         Line = "%s:%s" % (a['uid'], Pass)
+         Line = Sanitize(Line) + "\n"
+         F.write("%s" % (Line))

 
 

-      try:
-         OldMask = os.umask(0077);
-         File = os.path.join(GlobalDir, 'userkeys', User)
-         F = open(File + ".tmp","w",0600);
-         os.umask(OldMask);
+   except:
+      Die(File, None, F)
+      raise

 
 

-         for I in x[1]["sshRSAAuthKey"]:
-            MultipleLine = "%s" % I
-            MultipleLine = Sanitize(MultipleLine) + "\n"
-            F.write(MultipleLine)
+# Generate the rtcPassword list
+def GenRtcPassword(accounts, File):
+   F = None
+   try:
+      OldMask = os.umask(0077)
+      F = open(File, "w", 0600)
+      os.umask(OldMask)

 
 

-         Done(File,F,None);
-         userfiles.append(os.path.basename(File))
+      for a in accounts:
+         if not 'rtcPassword' in a: continue
+         if not a.pw_active(): continue

 
 

-      # Oops, something unspeakable happened.
-      except IOError:
-          Die(File,F,None)
-          Die(masterFileName,masterFile,None)
-          raise;
-
-   return userfiles
-
-def GenSSHtarballs(userlist, SSHFiles, grouprevmap, target):
-   OldMask = os.umask(0077);
-   tf = tarfile.open(name=os.path.join(GlobalDir, 'ssh-keys-%s.tar.gz' % CurrentHost), mode='w:gz')
-   os.umask(OldMask);
-   for f in userlist.keys():
-      if f not in SSHFiles:
+         Line = "%s@debian.org:%s:rtc.debian.org:AUTHORIZED" % (a['uid'], str(a['rtcPassword']))
+         Line = Sanitize(Line) + "\n"
+         F.write("%s" % (Line))
+
+   except:
+      Die(File, None, F)
+      raise
+
+def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
+   OldMask = os.umask(0077)
+   tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
+   os.umask(OldMask)
+   for f in userlist:
+      if f not in ssh_userkeys:

          continue
       # If we're not exporting their primary group, don't export
       # the key and warn
          continue
       # If we're not exporting their primary group, don't export
       # the key and warn


@@ -370,10 +467,21 @@ def GenSSHtarballs(userlist, SSHFiles, grouprevmap, target):
             pass
 
       if grname is None:
             pass
 
       if grname is None:

-         print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, CurrentHost, userlist[f])
+         print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f])

          continue
 
          continue
 

-      to = tf.gettarinfo(os.path.join(GlobalDir, 'userkeys', f), f)
+      lines = []
+      for line in ssh_userkeys[f]:
+         if line.startswith("allowed_hosts=") and ' ' in line:
+            machines, line = line.split('=', 1)[1].split(' ', 1)
+            if current_host not in machines.split(','):
+               continue # skip this key
+         lines.append(line)
+      if not lines:
+         continue # no keys for this host
+      contents = "\n".join(lines) + "\n"
+
+      to = tarfile.TarInfo(name=f)

       # These will only be used where the username doesn't
       # exist on the target system for some reason; hence,
       # in those cases, the safest thing is for the file to
       # These will only be used where the username doesn't
       # exist on the target system for some reason; hence,
       # in those cases, the safest thing is for the file to


@@ -389,26 +497,29 @@ def GenSSHtarballs(userlist, SSHFiles, grouprevmap, target):
       to.uname = f
       to.gname = grname
       to.mode  = 0400
       to.uname = f
       to.gname = grname
       to.mode  = 0400

-      tf.addfile(to, file(os.path.join(GlobalDir, 'userkeys', f)))
+      to.mtime = int(time.time())
+      to.size = len(contents)
+
+      tf.addfile(to, StringIO(contents))

 
    tf.close()
 
    tf.close()

-   os.rename(os.path.join(GlobalDir, 'ssh-keys-%s.tar.gz' % CurrentHost), target)
+   os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)

 
 # add a list of groups to existing groups,
 # including all subgroups thereof, recursively.
 # basically this proceduces the transitive hull of the groups in
 # addgroups.
 
 # add a list of groups to existing groups,
 # including all subgroups thereof, recursively.
 # basically this proceduces the transitive hull of the groups in
 # addgroups.

-def addGroups(existingGroups, newGroups, uid):
+def addGroups(existingGroups, newGroups, uid, current_host):

    for group in newGroups:
       # if it's a <group>@host, split it and verify it's on the current host.
       s = group.split('@', 1)
    for group in newGroups:
       # if it's a <group>@host, split it and verify it's on the current host.
       s = group.split('@', 1)

-      if len(s) == 2 and s[1] != CurrentHost:
-         continue;
+      if len(s) == 2 and s[1] != current_host:
+         continue

       group = s[0]
 
       # let's see if we handled this group already
       if group in existingGroups:
       group = s[0]
 
       # let's see if we handled this group already
       if group in existingGroups:

-        continue
+         continue

 
       if not GroupIDMap.has_key(group):
          print "Group", group, "does not exist but", uid, "is in it"
 
       if not GroupIDMap.has_key(group):
          print "Group", group, "does not exist but", uid, "is in it"


@@ -417,796 +528,971 @@ def addGroups(existingGroups, newGroups, uid):
       existingGroups.append(group)
 
       if SubGroupMap.has_key(group):
       existingGroups.append(group)
 
       if SubGroupMap.has_key(group):

-         addGroups(existingGroups, SubGroupMap[group], uid)
+         addGroups(existingGroups, SubGroupMap[group], uid, current_host)

 
 # Generate the group list
 
 # Generate the group list

-def GenGroup(l,File):
-  grouprevmap = {}
-  F = None;
-  try:
-   F = open(File + ".tdb.tmp","w");
-
-   # Generate the GroupMap
-   GroupMap = {};
-   for x in GroupIDMap.keys():
-      GroupMap[x] = [];
-
-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   # Sort them into a list of groups having a set of users
-   for x in PasswdAttrs:
-      uid = GetAttr(x,"uid")
-      if x[1].has_key("uidNumber") == 0 or IsInGroup(x) == 0:
-         continue;
-      if x[1].has_key("supplementaryGid") == 0:
-         continue;
-
-      supgroups=[]
-      addGroups(supgroups, x[1]["supplementaryGid"], uid)
-      for g in supgroups:
-         GroupMap[g].append(uid);
-
-   # Output the group file.
-   J = 0;
-   for x in GroupMap.keys():
-      grouprevmap[GroupIDMap[x]] = x
-      if GroupIDMap.has_key(x) == 0:
-         continue;
-      Line = "%s:x:%u:" % (x,GroupIDMap[x]);
-      Comma = '';
-      for I in GroupMap[x]:
-        Line = Line + ("%s%s" % (Comma,I));
-        Comma = ',';
-      Line = Sanitize(Line) + "\n";
-      F.write("0%u %s" % (J,Line));
-      F.write(".%s %s" % (x,Line));
-      F.write("=%u %s" % (GroupIDMap[x],Line));
-      J = J + 1;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,None,F);
-   raise;
-  Done(File,None,F);
-
-  return grouprevmap
-
-# Generate the email forwarding list
-def GenForward(l,File):
-  F = None;
-  try:
-   OldMask = os.umask(0022);
-   F = open(File + ".tmp","w",0644);
-   os.umask(OldMask);
+def GenGroup(accounts, File, current_host):
+   grouprevmap = {}
+   F = None
+   try:
+      F = open(File + ".tdb.tmp", "w")
+     
+      # Generate the GroupMap
+      GroupMap = {}
+      for x in GroupIDMap:
+         GroupMap[x] = []
+      GroupHasPrimaryMembers = {}
+
+      # Sort them into a list of groups having a set of users
+      for a in accounts:
+         GroupHasPrimaryMembers[ a['gidNumber'] ] = True
+         if not 'supplementaryGid' in a: continue
+
+         supgroups=[]
+         addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
+         for g in supgroups:
+            GroupMap[g].append(a['uid'])
+
+      # Output the group file.
+      J = 0
+      for x in GroupMap.keys():
+         if not x in GroupIDMap:
+            continue

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
+         if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
+            continue

 
 

-   # Write out the email address for each user
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+         grouprevmap[GroupIDMap[x]] = x

 
 

-      if x[1].has_key("emailForward") == 0 or IsInGroup(x) == 0:
-         continue;
+         Line = "%s:x:%u:" % (x, GroupIDMap[x])
+         Comma = ''
+         for I in GroupMap[x]:
+            Line = Line + ("%s%s" % (Comma, I))
+            Comma = ','
+         Line = Sanitize(Line) + "\n"
+         F.write("0%u %s" % (J, Line))
+         F.write(".%s %s" % (x, Line))
+         F.write("=%u %s" % (GroupIDMap[x], Line))
+         J = J + 1
+  
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, None, F)
+      raise
+   Done(File, None, F)
+  
+   return grouprevmap
+
+def CheckForward(accounts):
+   for a in accounts:
+      if not 'emailForward' in a: continue
+
+      delete = False

 
       # Do not allow people to try to buffer overflow busted parsers
 
       # Do not allow people to try to buffer overflow busted parsers

-      if len(GetAttr(x,"emailForward")) > 200:
-         continue;
-
+      if len(a['emailForward']) > 200: delete = True

       # Check the forwarding address
       # Check the forwarding address

-      if EmailCheck.match(GetAttr(x,"emailForward")) == None:
-         continue;
-      Line = "%s: %s" % (GetAttr(x,"uid"),GetAttr(x,"emailForward"));
-      Line = Sanitize(Line) + "\n";
-      F.write(Line);
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
-
-def GenAllForward(l,File):
-  Fdb = None;
-  try:
-   OldMask = os.umask(0022);
-   Fdb = os.popen("cdbmake %s %s.tmp"%(File,File),"w");
-   os.umask(OldMask);
+      elif EmailCheck.match(a['emailForward']) is None: delete = True

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
+      if delete:
+         a.delete_mailforward()

 
 

-   # Write out the email address for each user
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+# Generate the email forwarding list
+def GenForward(accounts, File):
+   F = None
+   try:
+      OldMask = os.umask(0022)
+      F = open(File + ".tmp", "w", 0644)
+      os.umask(OldMask)

 
 

-      if x[1].has_key("emailForward") == 0:
-         continue;
+      for a in accounts:
+         if not 'emailForward' in a: continue
+         Line = "%s: %s" % (a['uid'], a['emailForward'])
+         Line = Sanitize(Line) + "\n"
+         F.write(Line)

 
 

-      # Do not allow people to try to buffer overflow busted parsers
-      Forward = GetAttr(x,"emailForward");
-      if len(Forward) > 200:
-         continue;
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-      # Check the forwarding address
-      if EmailCheck.match(Forward) == None:
-         continue;
-
-      User = GetAttr(x,"uid");
-      Fdb.write("+%d,%d:%s->%s\n"%(len(User),len(Forward),User,Forward));
-   Fdb.write("\n");
-  # Oops, something unspeakable happened.
-  except:
-    Fdb.close();
-    raise;
-  if Fdb.close() != None:
-    raise "cdbmake gave an error";
+def GenCDB(accounts, File, key):
+   Fdb = None
+   try:
+      OldMask = os.umask(0022)
+      # nothing else does the fsync stuff, so why do it here?
+      prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else ''
+      Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w")
+      os.umask(OldMask)
+
+      # Write out the email address for each user
+      for a in accounts:
+         if not key in a: continue
+         value = a[key]
+         user = a['uid']
+         Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
+
+      Fdb.write("\n")
+   # Oops, something unspeakable happened.
+   except:
+      Fdb.close()
+      raise
+   if Fdb.close() != None:
+      raise "cdbmake gave an error"
+
+def GenDBM(accounts, File, key):
+   Fdb = None
+   OldMask = os.umask(0022)
+   fn = os.path.join(File).encode('ascii', 'ignore')
+   try:
+      posix.remove(fn)
+   except:
+      pass
+
+   try:
+      Fdb = dbm.open(fn, "c")
+      os.umask(OldMask)
+
+      # Write out the email address for each user
+      for a in accounts:
+         if not key in a: continue
+         value = a[key]
+         user = a['uid']
+         Fdb[user] = value
+
+      Fdb.close()
+   except:
+      # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
+      os.remove(File + ".db")
+      raise
+   # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
+   os.rename (File + ".db", File)

 
 # Generate the anon XEarth marker file
 
 # Generate the anon XEarth marker file

-def GenMarkers(l,File):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+def GenMarkers(accounts, File):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
+      # Write out the position for each user
+      for a in accounts:
+         if not ('latitude' in a and 'longitude' in a): continue
+         try:
+            Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True))
+            Line = Sanitize(Line) + "\n"
+            F.write(Line)
+         except:
+            pass
+  
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-   # Write out the position for each user
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+# Generate the debian-private subscription list
+def GenPrivate(accounts, File):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")

 
 

-      if x[1].has_key("latitude") == 0 or x[1].has_key("longitude") == 0:
-         continue;
-      try:
-         Line = "%8s %8s \"\""%(DecDegree(GetAttr(x,"latitude"),1),DecDegree(GetAttr(x,"longitude"),1));
-         Line = Sanitize(Line) + "\n";
-         F.write(Line);
-      except:
-         pass;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
+      # Write out the position for each user
+      for a in accounts:
+         if not a.is_active_user(): continue
+         if a.is_guest_account(): continue
+         if not 'privateSub' in a: continue
+         try:
+            Line = "%s"%(a['privateSub'])
+            Line = Sanitize(Line) + "\n"
+            F.write(Line)
+         except:
+            pass
+  
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-# Generate the debian-private subscription list
-def GenPrivate(l,File):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+# Generate a list of locked accounts
+def GenDisabledAccounts(accounts, File):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")
+      disabled_accounts = []
+
+      # Fetch all the users
+      for a in accounts:
+         if a.pw_active(): continue
+         Line = "%s:%s" % (a['uid'], "Account is locked")
+         disabled_accounts.append(a)
+         F.write(Sanitize(Line) + "\n")

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)
+   return disabled_accounts

 
 

-   # Write out the position for each user
-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+# Generate the list of local addresses that refuse all mail
+def GenMailDisable(accounts, File):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")

 
 

-      if x[1].has_key("privateSub") == 0:
-         continue;
+      for a in accounts:
+         if not 'mailDisableMessage' in a: continue
+         Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
+         Line = Sanitize(Line) + "\n"
+         F.write(Line)

 
 

-      # If the account is locked, do not write it
-      if (GetAttr(x,"userPassword").find("*LK*") != -1) \
-             or GetAttr(x,"userPassword").startswith("!"):
-         continue;
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-      # If the account has no PGP key, do not write it
-      if x[1].has_key("keyFingerPrint") == 0:
-         continue;
+# Generate a list of uids that should have boolean affects applied
+def GenMailBool(accounts, File, key):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")

 
 

-      # Must be in the Debian group (yuk, hard coded for now)
-      if GetAttr(x,"gidNumber") != "800":
-         continue;
+      for a in accounts:
+         if not key in a: continue
+         if not a[key] == 'TRUE': continue
+         Line = "%s"%(a['uid'])
+         Line = Sanitize(Line) + "\n"
+         F.write(Line)

 
 

-      try:
-         Line = "%s"%(GetAttr(x,"privateSub"));
-         Line = Sanitize(Line) + "\n";
-         F.write(Line);
-      except:
-         pass;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-# Generate a list of locked accounts
-def GenDisabledAccounts(l,File):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+# Generate a list of hosts for RBL or whitelist purposes.
+def GenMailList(accounts, File, key):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   global disabledusers
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   I = 0;
-   for x in PasswdAttrs:
-      if x[1].has_key("uidNumber") == 0:
-         continue;
-
-      Pass = GetAttr(x,"userPassword");
-      Line = ""
-      # *LK* is the reference value for a locked account
-      # password starting with ! is also a locked account
-      if Pass.find("*LK*") != -1 or Pass.startswith("!"):
-         # Format is <login>:<reason>
-         Line = "%s:%s" % (GetAttr(x,"uid"), "Account is locked")
-
-      if Line != "":
-         F.write(Sanitize(Line) + "\n")
+      if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$')
+      else:                      validregex = re.compile('^[-\w.]+$')

 
 

-      disabledusers.append(x)
+      for a in accounts:
+         if not key in a: continue

 
 

-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
+         filtered = filter(lambda z: validregex.match(z), a[key])
+         if len(filtered) == 0: continue
+         if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered)
+         line = a['uid'] + ': ' + ' : '.join(filtered)
+         line = Sanitize(line) + "\n"
+         F.write(line)

 
 

-# Generate the list of local addresses that refuse all mail
-def GenMailDisable(l,File):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
+def isRoleAccount(account):
+   return 'debianRoleAccount' in account['objectClass']

 
 

-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+# Generate the DNS Zone file
+def GenDNS(accounts, File):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")

 
 

-      Reason = None
+      # Fetch all the users
+      RRs = {}

 
 

-      if x[1].has_key("mailDisableMessage"):
-         Reason = GetAttr(x,"mailDisableMessage")
-      else:
-         continue
+      # Write out the zone file entry for each user
+      for a in accounts:
+         if not 'dnsZoneEntry' in a: continue
+         if not a.is_active_user() and not isRoleAccount(a): continue
+         if a.is_guest_account(): continue

 
 

-      # Must be in the Debian group (yuk, hard coded for now)
-      if GetAttr(x,"gidNumber") != "800":
-         continue;
+         try:
+            F.write("; %s\n"%(a.email_address()))
+            for z in a["dnsZoneEntry"]:
+               Split = z.lower().split()
+               if Split[1].lower() == 'in':
+                  Line = " ".join(Split) + "\n"
+                  F.write(Line)
+
+                  Host = Split[0] + DNSZone
+                  if BSMTPCheck.match(Line) != None:
+                     F.write("; Has BSMTP\n")
+
+                  # Write some identification information
+                  if not RRs.has_key(Host):
+                     if Split[2].lower() in ["a", "aaaa"]:
+                        Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address())
+                        for y in a["keyFingerPrint"]:
+                           Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y))
+                           F.write(Line)
+                        RRs[Host] = 1
+               else:
+                  Line = "; Err %s"%(str(Split))
+                  F.write(Line)
+
+            F.write("\n")
+         except Exception, e:
+            F.write("; Errors:\n")
+            for line in str(e).split("\n"):
+               F.write("; %s\n"%(line))
+            pass
+  
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)
+
+def is_ipv6_addr(i):
+   try:
+      socket.inet_pton(socket.AF_INET6, i)
+   except socket.error:
+      return False
+   return True

 
 

-      try:
-         Line = "%s: %s"%(GetAttr(x,"uid"),Reason);
-         Line = Sanitize(Line) + "\n";
-         F.write(Line);
-      except:
-         pass;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
+def ExtractDNSInfo(x):

 
 

-# Generate a list of uids that should have boolean affects applied
-def GenMailBool(l,File,Key):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+   TTLprefix="\t"
+   if 'dnsTTL' in x[1]:
+      TTLprefix="%s\t"%(x[1]["dnsTTL"][0])

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
+   DNSInfo = []
+   if x[1].has_key("ipHostNumber"):
+      for I in x[1]["ipHostNumber"]:
+         if is_ipv6_addr(I):
+            DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I))
+         else:
+            DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I))

 
 

-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+   Algorithm = None

 
 

-      Reason = None
+   if 'sshRSAHostKey' in x[1]:
+      for I in x[1]["sshRSAHostKey"]:
+         Split = I.split()
+         if Split[0] == 'ssh-rsa':
+            Algorithm = 1
+         if Split[0] == 'ssh-dss':
+            Algorithm = 2
+         if Split[0] == 'ssh-ed25519':
+            Algorithm = 4
+         if Algorithm == None:
+            continue
+         Fingerprint = hashlib.new('sha1', base64.decodestring(Split[1])).hexdigest()
+         DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint))
+         Fingerprint = hashlib.new('sha256', base64.decodestring(Split[1])).hexdigest()
+         DNSInfo.append("%sIN\tSSHFP\t%u 2 %s" % (TTLprefix, Algorithm, Fingerprint))
+
+   if 'architecture' in x[1]:
+      Arch = GetAttr(x, "architecture")
+      Mach = ""
+      if x[1].has_key("machine"):
+         Mach = " " + GetAttr(x, "machine")
+      DNSInfo.append("%sIN\tHINFO\t\"%s%s\" \"%s\"" % (TTLprefix, Arch, Mach, "Debian GNU/Linux"))
+
+   if x[1].has_key("mXRecord"):
+      for I in x[1]["mXRecord"]:
+         if I in MX_remap:
+            for e in MX_remap[I]:
+               DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, e))
+         else:
+            DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I))

 
 

-      if x[1].has_key(Key) == 0:
-         continue
+   return DNSInfo

 
 

-      # Must be in the Debian group (yuk, hard coded for now)
-      if GetAttr(x,"gidNumber") != "800":
-         continue
+# Generate the DNS records
+def GenZoneRecords(host_attrs, File):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")

 
 

-      if GetAttr(x,Key) != "TRUE":
-         continue
+      # Fetch all the hosts
+      for x in host_attrs:
+         if x[1].has_key("hostname") == 0:
+            continue

 
 

-      try:
-         Line = "%s"%(GetAttr(x,"uid"));
-         Line = Sanitize(Line) + "\n";
-         F.write(Line);
-      except:
-         pass;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
+         if IsDebianHost.match(GetAttr(x, "hostname")) is None:
+            continue

 
 

-# Generate a list of hosts for RBL or whitelist purposes.
-def GenMailList(l,File,Key):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+         DNSInfo = ExtractDNSInfo(x)
+         start = True
+         for Line in DNSInfo:
+            if start == True:
+               Line = "%s.\t%s" % (GetAttr(x, "hostname"), Line)
+               start = False
+            else:
+               Line = "\t\t\t%s" % (Line)
+
+            F.write(Line + "\n")
+
+        # this would write sshfp lines for services on machines
+        # but we can't yet, since some are cnames and we'll make
+        # an invalid zonefile
+        #
+        # for i in x[1].get("purpose", []):
+        #    m = PurposeHostField.match(i)
+        #    if m:
+        #       m = m.group(1)
+        #       # we ignore [[*..]] entries
+        #       if m.startswith('*'):
+        #          continue
+        #       if m.startswith('-'):
+        #          m = m[1:]
+        #       if m:
+        #          if not m.endswith(HostDomain):
+        #             continue
+        #          if not m.endswith('.'):
+        #             m = m + "."
+        #          for Line in DNSInfo:
+        #             if isSSHFP.match(Line):
+        #                Line = "%s\t%s" % (m, Line)
+        #                F.write(Line + "\n")
+
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
+# Generate the BSMTP file
+def GenBSMTP(accounts, File, HomePrefix):
+   F = None
+   try:
+      F = open(File + ".tmp", "w")
+     
+      # Write out the zone file entry for each user
+      for a in accounts:
+         if not 'dnsZoneEntry' in a: continue
+         if not a.is_active_user(): continue

 
 

-   for x in PasswdAttrs:
-      if IsRetired(x):
-         continue
+         try:
+            for z in a["dnsZoneEntry"]:
+               Split = z.lower().split()
+               if Split[1].lower() == 'in':
+                  for y in range(0, len(Split)):
+                     if Split[y] == "$":
+                        Split[y] = "\n\t"
+                  Line = " ".join(Split) + "\n"
+     
+                  Host = Split[0] + DNSZone
+                  if BSMTPCheck.match(Line) != None:
+                      F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
+                                  a['uid'], HomePrefix, a['uid'], Host))
+     
+         except:
+            F.write("; Errors\n")
+            pass
+  
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)
+  
+def HostToIP(Host, mapped=True):

 
 

-      Reason = None
+   IPAdresses = []

 
 

-      if x[1].has_key(Key) == 0:
-         continue
+   if Host[1].has_key("ipHostNumber"):
+      for addr in Host[1]["ipHostNumber"]:
+         IPAdresses.append(addr)
+         if not is_ipv6_addr(addr) and mapped == "True":
+            IPAdresses.append("::ffff:"+addr)

 
 

-      # Must be in the Debian group (yuk, hard coded for now)
-      if GetAttr(x,"gidNumber") != "800":
-         continue
+   return IPAdresses

 
 

-      try:
-         found = 0
-         Line = None
-         for z in x[1][Key]:
-             if Key == "mailWhitelist":
-                 if re.match('^[-\w.]+(/[\d]+)?$',z) == None:
-                     continue
-             else:
-                 if re.match('^[-\w.]+$',z) == None:
-                     continue
-             if found == 0:
-                 found = 1
-                 Line = GetAttr(x,"uid")
-             else:
-                 Line += " "
-             Line += ": " + z
-             if Key == "mailRHSBL":
-                 Line += "/$sender_address_domain"
-
-         if Line != None:
-             Line = Sanitize(Line) + "\n";
-             F.write(Line);
-      except:
-         pass;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
-
-def isRoleAccount(pwEntry):
-   if not pwEntry.has_key("objectClass"):
-      raise "pwEntry has no objectClass"
-   oc =  pwEntry['objectClass']
+# Generate the ssh known hosts file
+def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
+   F = None

    try:
    try:

-      i = oc.index('debianRoleAccount')
-      return True
-   except ValueError:
-      return False
+      OldMask = os.umask(0022)
+      F = open(File + ".tmp", "w", 0644)
+      os.umask(OldMask)
+     
+      for x in host_attrs:
+         if x[1].has_key("hostname") == 0 or \
+            x[1].has_key("sshRSAHostKey") == 0:
+            continue
+         Host = GetAttr(x, "hostname")
+         HostNames = [ Host ]
+         if Host.endswith(HostDomain):
+            HostNames.append(Host[:-(len(HostDomain) + 1)])
+     
+         # in the purpose field [[host|some other text]] (where some other text is optional)
+         # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
+         # file.  But so that we don't have to add everything we link we can add an asterisk
+         # and say [[*... to ignore it.  In order to be able to add stuff to ssh without
+         # http linking it we also support [[-hostname]] entries.
+         for i in x[1].get("purpose", []):
+            m = PurposeHostField.match(i)
+            if m:
+               m = m.group(1)
+               # we ignore [[*..]] entries
+               if m.startswith('*'):
+                  continue
+               if m.startswith('-'):
+                  m = m[1:]
+               if m:
+                  HostNames.append(m)
+                  if m.endswith(HostDomain):
+                     HostNames.append(m[:-(len(HostDomain) + 1)])
+     
+         for I in x[1]["sshRSAHostKey"]:
+            if mode and mode == 'authorized_keys':
+               hosts = HostToIP(x)
+               if 'sshdistAuthKeysHost' in x[1]:
+                  hosts += x[1]['sshdistAuthKeysHost']
+               clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host)
+               clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand)
+               Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I)
+            else:
+               Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I)
+            Line = Sanitize(Line) + "\n"
+            F.write(Line)
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)

 
 

-# Generate the DNS Zone file
-def GenDNS(l,File,HomePrefix):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+# Generate the debianhosts file (list of all IP addresses)
+def GenHosts(host_attrs, File):
+   F = None
+   try:
+      OldMask = os.umask(0022)
+      F = open(File + ".tmp", "w", 0644)
+      os.umask(OldMask)
+     
+      seen = set()

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   # Write out the zone file entry for each user
-   for x in PasswdAttrs:
-      if x[1].has_key("dnsZoneEntry") == 0:
-         continue;
-
-      # If the account has no PGP key, do not write it
-      if x[1].has_key("keyFingerPrint") == 0 and not isRoleAccount(x[1]):
-         continue;
-      try:
-         F.write("; %s\n"%(EmailAddress(x)));
-         for z in x[1]["dnsZoneEntry"]:
-            Split = z.lower().split()
-            if Split[1].lower() == 'in':
-               for y in range(0,len(Split)):
-                  if Split[y] == "$":
-                     Split[y] = "\n\t";
-               Line = " ".join(Split) + "\n";
-               F.write(Line);
-
-               Host = Split[0] + DNSZone;
-               if BSMTPCheck.match(Line) != None:
-                   F.write("; Has BSMTP\n");
-
-               # Write some identification information
-               if Split[2].lower() == "a":
-                  Line = "%s IN TXT \"%s\"\n"%(Split[0],EmailAddress(x));
-                  for y in x[1]["keyFingerPrint"]:
-                     Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0],FormatPGPKey(y));
-                  F.write(Line);
-            else:
-               Line = "; Err %s"%(str(Split));
-               F.write(Line);
-
-         F.write("\n");
-      except:
-         F.write("; Errors\n");
-         pass;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
-
-# Generate the DNS SSHFP records
-def GenSSHFP(l,File,HomePrefix):
-  F = None
-  try:
-   F = open(File + ".tmp","w")
+      for x in host_attrs:
+
+         if IsDebianHost.match(GetAttr(x, "hostname")) is None:
+            continue
+
+         if not 'ipHostNumber' in x[1]:
+            continue

 
 

+         addrs = x[1]["ipHostNumber"]
+         for addr in addrs:
+            if addr not in seen:
+               seen.add(addr)
+               addr = Sanitize(addr) + "\n"
+               F.write(addr)
+
+   # Oops, something unspeakable happened.
+   except:
+      Die(File, F, None)
+      raise
+   Done(File, F, None)
+
+def replaceTree(src, dst_basedir):
+   bn = os.path.basename(src)
+   dst = os.path.join(dst_basedir, bn)
+   safe_rmtree(dst)
+   shutil.copytree(src, dst)
+
+def GenKeyrings(OutDir):
+   for k in Keyrings:
+      if os.path.isdir(k):
+         replaceTree(k, OutDir)
+      else:
+         shutil.copy(k, OutDir)
+
+
+def get_accounts(ldap_conn):
+   # Fetch all the users
+   passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\
+                   ["uid", "uidNumber", "gidNumber", "supplementaryGid",\
+                    "gecos", "loginShell", "userPassword", "shadowLastChange",\
+                    "shadowMin", "shadowMax", "shadowWarning", "shadowInactive",
+                    "shadowExpire", "emailForward", "latitude", "longitude",\
+                    "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\
+                    "keyFingerPrint", "privateSub", "mailDisableMessage",\
+                    "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
+                    "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
+                    "mailContentInspectionAction", "webPassword", "rtcPassword",\
+                    "bATVToken"])
+
+   if passwd_attrs is None:
+      raise UDEmptyList, "No Users"
+   accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs)
+   accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower()))
+
+   return accounts
+
+def get_hosts(ldap_conn):

    # Fetch all the hosts
    # Fetch all the hosts

-   global HostAttrs
+   HostAttrs    = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
+                   ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
+                    "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"])
+

    if HostAttrs == None:
    if HostAttrs == None:

-      raise "No Hosts"
+      raise UDEmptyList, "No Hosts"
+
+   HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))
+
+   return HostAttrs

 
 

-   for x in HostAttrs:
-      if x[1].has_key("hostname") == 0 or \
-         x[1].has_key("sshRSAHostKey") == 0:
+
+def make_ldap_conn():
+   # Connect to the ldap server
+   l = connectLDAP()
+   # for testing purposes it's sometimes useful to pass username/password
+   # via the environment
+   if 'UD_CREDENTIALS' in os.environ:
+      Pass = os.environ['UD_CREDENTIALS'].split()
+   else:
+      F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r")
+      Pass = F.readline().strip().split(" ")
+      F.close()
+   l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1])
+
+   return l
+
+
+
+def setup_group_maps(l):
+   # Fetch all the groups
+   group_id_map = {}
+   subgroup_map = {}
+   attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
+                     ["gid", "gidNumber", "subGroup"])
+
+   # Generate the subgroup_map and group_id_map
+   for x in attrs:
+      if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":

          continue
          continue

-      Host = GetAttr(x,"hostname");
-      Algorithm = None
-      for I in x[1]["sshRSAHostKey"]:
-         Split = I.split()
-         if Split[0] == 'ssh-rsa':
-            Algorithm = 1
-         if Split[0] == 'ssh-dss':
-            Algorithm = 2
-         if Algorithm == None:
-            continue
-         Fingerprint = sha.new(base64.decodestring(Split[1])).hexdigest()
-         Line = "%s. IN SSHFP %u 1 %s" % (Host,Algorithm,Fingerprint)
-         Line = Sanitize(Line) + "\n"
-         F.write(Line)
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None)
-   raise;
-  Done(File,F,None)
+      if x[1].has_key("gidNumber") == 0:
+         continue
+      group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
+      if x[1].has_key("subGroup") != 0:
+         subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])
+
+   global SubGroupMap
+   global GroupIDMap
+   SubGroupMap = subgroup_map
+   GroupIDMap = group_id_map
+
+def generate_all(global_dir, ldap_conn):
+   accounts = get_accounts(ldap_conn)
+   host_attrs = get_hosts(ldap_conn)
+
+   global_dir += '/'
+   # Generate global things
+   accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
+
+   accounts = filter(lambda x: not IsRetired(x), accounts)
+
+   CheckForward(accounts)
+
+   GenMailDisable(accounts, global_dir + "mail-disable")
+   GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
+   GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
+   GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
+   GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
+   GenPrivate(accounts, global_dir + "debian-private")
+   GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
+   GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
+   GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
+   GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
+   GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
+   GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
+   GenWebPassword(accounts, global_dir + "web-passwords")
+   GenRtcPassword(accounts, global_dir + "rtc-passwords")
+   GenKeyrings(global_dir)

 
 

-# Generate the BSMTP file
-def GenBSMTP(l,File,HomePrefix):
-  F = None;
-  try:
-   F = open(File + ".tmp","w");
+   # Compatibility.
+   GenForward(accounts, global_dir + "forward-alias")

 
 

-   # Fetch all the users
-   global PasswdAttrs;
-   if PasswdAttrs == None:
-      raise "No Users";
-
-   # Write out the zone file entry for each user
-   for x in PasswdAttrs:
-      if x[1].has_key("dnsZoneEntry") == 0:
-         continue;
-
-      # If the account has no PGP key, do not write it
-      if x[1].has_key("keyFingerPrint") == 0:
-         continue;
-      try:
-         for z in x[1]["dnsZoneEntry"]:
-            Split = z.lower().split()
-            if Split[1].lower() == 'in':
-               for y in range(0,len(Split)):
-                  if Split[y] == "$":
-                     Split[y] = "\n\t";
-               Line = " ".join(Split) + "\n";
-
-               Host = Split[0] + DNSZone;
-               if BSMTPCheck.match(Line) != None:
-                   F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
-                               GetAttr(x,"uid"),HomePrefix,GetAttr(x,"uid"),Host));
-
-      except:
-         F.write("; Errors\n");
-         pass;
-
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
-
-# cache IP adresses
-HostToIPCache = {}
-def HostToIP(Host):
-    global HostToIPCache
-    if not Host in HostToIPCache:
-        IPAdressesT = None
-        try:
-            IPAdressesT = list(set([ (a[0],a[4][0]) for a in socket.getaddrinfo(Host, None)]))
-        except socket.gaierror, (code):
-            if code[0] != -2: raise
-        IPAdresses = []
-        if not IPAdressesT is None:
-            for addr in IPAdressesT:
-               if addr[0] == socket.AF_INET: IPAdresses += [addr[1], "::ffff:"+addr[1]]
-               else: IPAdresses += [addr[1]]
-        HostToIPCache[Host] = IPAdresses
-    return HostToIPCache[Host]
+   GenAllUsers(accounts, global_dir + 'all-accounts.json')
+   accounts = filter(lambda a: not a in accounts_disabled, accounts)

 
 

+   ssh_userkeys = GenSSHShadow(global_dir, accounts)
+   GenMarkers(accounts, global_dir + "markers")
+   GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
+   GenHosts(host_attrs, global_dir + "debianhosts")
+   GenSSHGitolite(accounts, host_attrs, global_dir + "ssh-gitolite")

 
 

-# Generate the ssh known hosts file
-def GenSSHKnown(l,File,mode=None):
-  F = None;
-  try:
-   OldMask = os.umask(0022);
-   F = open(File + ".tmp","w",0644);
-   os.umask(OldMask);
-
-   global HostAttrs
-   if HostAttrs == None:
-      raise "No Hosts";
-
-   for x in HostAttrs:
-      if x[1].has_key("hostname") == 0 or \
-         x[1].has_key("sshRSAHostKey") == 0:
-         continue;
-      Host = GetAttr(x,"hostname");
-      HostNames = [ Host ]
-      if Host.endswith(HostDomain):
-         HostNames.append(Host[:-(len(HostDomain)+1)])
-
-      # in the purpose field [[host|some other text]] (where some other text is optional)
-      # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
-      # file.  But so that we don't have to add everything we link we can add an asterisk
-      # and say [[*... to ignore it.  In order to be able to add stuff to ssh without
-      # http linking it we also support [[-hostname]] entries.
-      for i in x[1].get("purpose",[]):
-         m = PurposeHostField.match(i)
-         if m:
-            m = m.group(1)
-            # we ignore [[*..]] entries
-            if m.startswith('*'):
-               continue;
-            if m.startswith('-'):
-               m = m[1:]
-            if m:
-               HostNames.append(m)
-               if m.endswith(HostDomain):
-                  HostNames.append(m[:-(len(HostDomain)+1)])
+   GenDNS(accounts, global_dir + "dns-zone")
+   GenZoneRecords(host_attrs, global_dir + "dns-sshfp")

 
 

-      for I in x[1]["sshRSAHostKey"]:
-         if mode and mode == 'authorized_keys':
-            #Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (Host, ",".join(HNames + HostToIP(Host)), I)
-            Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding %s' % (Host,I)
-         else:
-            Line = "%s %s" %(",".join(HostNames + HostToIP(Host)), I);
-         Line = Sanitize(Line) + "\n";
-         F.write(Line);
-  # Oops, something unspeakable happened.
-  except:
-   Die(File,F,None);
-   raise;
-  Done(File,F,None);
+   setup_group_maps(ldap_conn)

 
 

-# Generate the debianhosts file (list of all IP addresses)
-def GenHosts(l,File):
-  F = None
-  try:
-    OldMask = os.umask(0022)
-    F = open(File + ".tmp","w",0644)
-    os.umask(OldMask)
-
-    # Fetch all the hosts
-    hostnames = l.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "hostname=*",
-                           ["hostname"])
-
-    if hostnames == None:
-       raise "No Hosts"
-
-    seen = set()
-    for x in hostnames:
-      host = GetAttr(x,"hostname", None)
-      if host:
-        addrs = []
-        try:
-          addrs += socket.getaddrinfo(host, None, socket.AF_INET)
-        except socket.error:
-          pass
-        try:
-          addrs += socket.getaddrinfo(host, None, socket.AF_INET6)
-        except socket.error:
-          pass
-
-        for addrinfo in addrs:
-          if addrinfo[0] in (socket.AF_INET, socket.AF_INET6):
-            addr = addrinfo[4][0]
-            if addr not in seen:
-              print >> F, addrinfo[4][0]
-              seen.add(addr)
-  # Oops, something unspeakable happened.
-  except:
-    Die(File,F,None)
-    raise
-  Done(File,F,None)
-
-def GenKeyrings(l,OutDir):
-  for k in Keyrings:
-    shutil.copy(k, OutDir)
-
-
-# Connect to the ldap server
-l = connectLDAP()
-F = open(PassDir+"/pass-"+pwd.getpwuid(os.getuid())[0],"r");
-Pass = F.readline().strip().split(" ")
-F.close();
-l.simple_bind_s("uid="+Pass[0]+","+BaseDn,Pass[1]);
-
-# Fetch all the groups
-GroupIDMap = {};
-Attrs = l.search_s(BaseDn,ldap.SCOPE_ONELEVEL,"gid=*",\
-                  ["gid","gidNumber","subGroup"]);
-
-# Generate the SubGroupMap and GroupIDMap
-for x in Attrs:
-   if x[1].has_key("gidNumber") == 0:
-      continue;
-   GroupIDMap[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]);
-   if x[1].has_key("subGroup") != 0:
-      SubGroupMap.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"]);
-
-# Fetch all the users
-PasswdAttrs = l.search_s(BaseDn,ldap.SCOPE_ONELEVEL,"uid=*",\
-                ["uid","uidNumber","gidNumber","supplementaryGid",\
-                 "gecos","loginShell","userPassword","shadowLastChange",\
-                 "shadowMin","shadowMax","shadowWarning","shadowInactive",
-                 "shadowExpire","emailForward","latitude","longitude",\
-                 "allowedHost","sshRSAAuthKey","dnsZoneEntry","cn","sn",\
-                 "keyFingerPrint","privateSub","mailDisableMessage",\
-                 "mailGreylisting","mailCallout","mailRBL","mailRHSBL",\
-                 "mailWhitelist", "sudoPassword", "objectClass", "accountStatus"]);
-# Fetch all the hosts
-HostAttrs    = l.search_s(HostBaseDn,ldap.SCOPE_ONELEVEL,"sshRSAHostKey=*",\
-                ["hostname","sshRSAHostKey","purpose"]);
-
-# Open the control file
-if len(sys.argv) == 1:
-   F = open(GenerateConf,"r");
-else:
-   F = open(sys.argv[1],"r")
-
-# Generate global things
-GlobalDir = GenerateDir+"/";
-SSHFiles = GenSSHShadow(l);
-GenAllForward(l,GlobalDir+"mail-forward.cdb");
-GenMarkers(l,GlobalDir+"markers");
-GenPrivate(l,GlobalDir+"debian-private");
-GenDisabledAccounts(l,GlobalDir+"disabled-accounts");
-GenSSHKnown(l,GlobalDir+"ssh_known_hosts");
-#GenSSHKnown(l,GlobalDir+"authorized_keys", 'authorized_keys');
-GenHosts(l,GlobalDir+"debianhosts");
-GenMailDisable(l,GlobalDir+"mail-disable");
-GenMailBool(l,GlobalDir+"mail-greylist","mailGreylisting");
-GenMailBool(l,GlobalDir+"mail-callout","mailCallout");
-GenMailList(l,GlobalDir+"mail-rbl","mailRBL");
-GenMailList(l,GlobalDir+"mail-rhsbl","mailRHSBL");
-GenMailList(l,GlobalDir+"mail-whitelist","mailWhitelist");
-GenKeyrings(l,GlobalDir);
-
-# Compatibility.
-GenForward(l,GlobalDir+"forward-alias");
-
-PasswdAttrs = filter(lambda x: not x in disabledusers, PasswdAttrs)
-
-while(1):
-   Line = F.readline();
-   if Line == "":
-      break;
-   Line = Line.strip()
-   if Line == "":
-      continue;
-   if Line[0] == '#':
-      continue;
-
-   Split = Line.split(" ")
-   OutDir = GenerateDir + '/' + Split[0] + '/';
-   try: os.mkdir(OutDir);
-   except: pass;
+   for host in host_attrs:
+      if not "hostname" in host[1]:
+         continue
+      generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
+
+def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
+   current_host = host[1]['hostname'][0]
+   OutDir = global_dir + current_host + '/'
+   if not os.path.isdir(OutDir):
+      os.mkdir(OutDir)

 
    # Get the group list and convert any named groups to numerics
 
    # Get the group list and convert any named groups to numerics

-   GroupList = {};
-   ExtraList = {};
-   for I in Split[2:]:
-      if I[0] == '[':
-         ExtraList[I] = None;
-         continue;
-      GroupList[I] = None;
-      if GroupIDMap.has_key(I):
-         GroupList[str(GroupIDMap[I])] = None;
-
-   Allowed = GroupList;
-   if Allowed == {}:
-     Allowed = None
-   CurrentHost = Split[0];
-
-   DoLink(GlobalDir,OutDir,"debianhosts");
-   DoLink(GlobalDir,OutDir,"ssh_known_hosts");
-   DoLink(GlobalDir,OutDir,"disabled-accounts")
-
-   sys.stdout.flush();
-   if ExtraList.has_key("[NOPASSWD]"):
-      userlist = GenPasswd(l,OutDir+"passwd",Split[1], "*");
+   GroupList = {}
+   for groupname in AllowedGroupsPreload.strip().split(" "):
+      GroupList[groupname] = True
+   if 'allowedGroups' in host[1]:
+      for groupname in host[1]['allowedGroups']:
+         GroupList[groupname] = True
+   for groupname in GroupList.keys():
+      if groupname in GroupIDMap:
+         GroupList[str(GroupIDMap[groupname])] = True
+
+   ExtraList = {}
+   if 'exportOptions' in host[1]:
+      for extra in host[1]['exportOptions']:
+         ExtraList[extra.upper()] = True
+
+   if GroupList != {}:
+      accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)
+
+   DoLink(global_dir, OutDir, "debianhosts")
+   DoLink(global_dir, OutDir, "ssh_known_hosts")
+   DoLink(global_dir, OutDir, "disabled-accounts")
+
+   sys.stdout.flush()
+   if 'NOPASSWD' in ExtraList:
+      userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*")

    else:
    else:

-      userlist = GenPasswd(l,OutDir+"passwd",Split[1], "x");
-   sys.stdout.flush();
-   grouprevmap = GenGroup(l,OutDir+"group");
-   GenShadowSudo(l, OutDir+"sudo-passwd", ExtraList.has_key("[UNTRUSTED]") or ExtraList.has_key("[NOPASSWD]"))
+      userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
+   sys.stdout.flush()
+   grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
+   GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)

 
    # Now we know who we're allowing on the machine, export
    # the relevant ssh keys
 
    # Now we know who we're allowing on the machine, export
    # the relevant ssh keys

-   GenSSHtarballs(userlist, SSHFiles, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'))
+   GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)

 
 

-   if ExtraList.has_key("[UNTRUSTED]"):
-     print "[UNTRUSTED] tag is obsolete and may be removed in the future."
-     continue;
-   if not ExtraList.has_key("[NOPASSWD]"):
-     GenShadow(l,OutDir+"shadow");
+   if not 'NOPASSWD' in ExtraList:
+      GenShadow(accounts, OutDir + "shadow")

 
    # Link in global things
 
    # Link in global things

-   if not ExtraList.has_key("[NOMARKERS]"):
-     DoLink(GlobalDir,OutDir,"markers");
-   DoLink(GlobalDir,OutDir,"mail-forward.cdb");
-   DoLink(GlobalDir,OutDir,"mail-disable");
-   DoLink(GlobalDir,OutDir,"mail-greylist");
-   DoLink(GlobalDir,OutDir,"mail-callout");
-   DoLink(GlobalDir,OutDir,"mail-rbl");
-   DoLink(GlobalDir,OutDir,"mail-rhsbl");
-   DoLink(GlobalDir,OutDir,"mail-whitelist");
+   if not 'NOMARKERS' in ExtraList:
+      DoLink(global_dir, OutDir, "markers")
+   DoLink(global_dir, OutDir, "mail-forward.cdb")
+   DoLink(global_dir, OutDir, "mail-forward.db")
+   DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
+   DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
+   DoLink(global_dir, OutDir, "mail-disable")
+   DoLink(global_dir, OutDir, "mail-greylist")
+   DoLink(global_dir, OutDir, "mail-callout")
+   DoLink(global_dir, OutDir, "mail-rbl")
+   DoLink(global_dir, OutDir, "mail-rhsbl")
+   DoLink(global_dir, OutDir, "mail-whitelist")
+   DoLink(global_dir, OutDir, "all-accounts.json")
+   GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
+   GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
+   GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
+   GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')
+   GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions')
+   GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions')

 
    # Compatibility.
 
    # Compatibility.

-   DoLink(GlobalDir,OutDir,"forward-alias");
-
-   if ExtraList.has_key("[DNS]"):
-      GenDNS(l,OutDir+"dns-zone",Split[1]);
-      GenSSHFP(l,OutDir+"dns-sshfp",Split[1])
+   DoLink(global_dir, OutDir, "forward-alias")
+
+   if 'DNS' in ExtraList:
+      DoLink(global_dir, OutDir, "dns-zone")
+      DoLink(global_dir, OutDir, "dns-sshfp")
+
+   if 'AUTHKEYS' in ExtraList:
+      DoLink(global_dir, OutDir, "authorized_keys")
+
+   if 'BSMTP' in ExtraList:
+      GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix)
+
+   if 'PRIVATE' in ExtraList:
+      DoLink(global_dir, OutDir, "debian-private")
+
+   if 'GITOLITE' in ExtraList:
+      DoLink(global_dir, OutDir, "ssh-gitolite")
+   if 'exportOptions' in host[1]:
+      for entry in host[1]['exportOptions']:
+         v = entry.split('=',1)
+         if v[0] != 'GITOLITE' or len(v) != 2: continue
+         options = v[1].split(',')
+         group = options.pop(0);
+         gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
+         if not 'nohosts' in options:
+            gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
+         else:
+            gitolite_hosts = []
+         command = None
+         for opt in options:
+            if opt.startswith('sshcmd='):
+               command = opt.split('=',1)[1]
+         GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)

 
 

-   if ExtraList.has_key("[BSMTP]"):
-      GenBSMTP(l,OutDir+"bsmtp",Split[1])
+   if 'WEB-PASSWORDS' in ExtraList:
+      DoLink(global_dir, OutDir, "web-passwords")

 
 

-   if ExtraList.has_key("[PRIVATE]"):
-      DoLink(GlobalDir,OutDir,"debian-private")
+   if 'RTC-PASSWORDS' in ExtraList:
+      DoLink(global_dir, OutDir, "rtc-passwords")

 
 

-   if ExtraList.has_key("[KEYRING]"):
+   if 'KEYRING' in ExtraList:

       for k in Keyrings:
       for k in Keyrings:

-        DoLink(GlobalDir,OutDir,os.path.basename(k))
+         bn = os.path.basename(k)
+         if os.path.isdir(k):
+            src = os.path.join(global_dir, bn)
+            replaceTree(src, OutDir)
+         else:
+            DoLink(global_dir, OutDir, bn)
+   else:
+      for k in Keyrings:
+         try:
+            bn = os.path.basename(k)
+            target = os.path.join(OutDir, bn)
+            if os.path.isdir(target):
+               safe_rmtree(dst)
+            else:
+               posix.remove(target)
+         except:
+            pass
+   DoLink(global_dir, OutDir, "last_update.trace")
+
+
+def getLastLDAPChangeTime(l):
+   mods = l.search_s('cn=log',
+         ldap.SCOPE_ONELEVEL,
+         '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))',
+         ['reqEnd'])
+
+   last = 0
+
+   # Sort the list by reqEnd
+   sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0])
+   # Take the last element in the array
+   last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0]
+
+   return last
+
+def getLastKeyringChangeTime():
+   krmod = 0
+   for k in Keyrings:
+      mt = os.path.getmtime(k)
+      if mt > krmod:
+         krmod = mt
+
+   return int(krmod)
+
+def getLastBuildTime(gdir):
+   cache_last_ldap_mod = 0
+   cache_last_unix_mod = 0
+   cache_last_run = 0
+
+   try:
+      fd = open(os.path.join(gdir, "last_update.trace"), "r")
+      cache_last_mod=fd.read().split()
+      try:
+         cache_last_ldap_mod = cache_last_mod[0]
+         cache_last_unix_mod = int(cache_last_mod[1])
+         cache_last_run = int(cache_last_mod[2])
+      except IndexError, ValueError:
+         pass
+      fd.close()
+   except IOError, e:
+      if e.errno == errno.ENOENT:
+         pass
+      else:
+         raise e
+
+   return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
+
+def mq_notify(options, message):
+   options.section = 'dsa-udgenerate'
+   options.config = '/etc/dsa/pubsub.conf'
+
+   config = Config(options)
+   conf = {
+      'rabbit_userid': config.username,
+      'rabbit_password': config.password,
+      'rabbit_virtual_host': config.vhost,
+      'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
+      'use_ssl': False
+   }
+
+   msg = {
+      'message': message,
+      'timestamp': int(time.time())
+   }
+   conn = None
+   try:
+      conn = Connection(conf=conf)
+      conn.topic_send(config.topic,
+            json.dumps(msg),
+            exchange_name=config.exchange,
+            timeout=5)
+   finally:
+      if conn:
+         conn.close()
+
+def ud_generate():
+   parser = optparse.OptionParser()
+   parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
+     help="Output directory.")
+   parser.add_option("-f", "--force", dest="force", action="store_true",
+     help="Force generation, even if no update to LDAP has happened.")
+
+   (options, args) = parser.parse_args()
+   if len(args) > 0:
+      parser.print_help()
+      sys.exit(1)
+
+   if options.generatedir is not None:
+      generate_dir = os.environ['UD_GENERATEDIR']
+   elif 'UD_GENERATEDIR' in os.environ:
+      generate_dir = os.environ['UD_GENERATEDIR']
+   else:
+      generate_dir = GenerateDir
+
+
+   lockf = os.path.join(generate_dir, 'ud-generate.lock')
+   lock = get_lock( lockf )
+   if lock is None:
+      sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
+      sys.exit(1)
+
+   l = make_ldap_conn()
+
+   time_started = int(time.time())
+   ldap_last_mod = getLastLDAPChangeTime(l)
+   unix_last_mod = getLastKeyringChangeTime()
+   cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)
+
+   need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
+
+   fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
+   if need_update or options.force:
+      msg = 'Update forced' if options.force else 'Update needed'
+      generate_all(generate_dir, l)
+      mq_notify(options, msg)
+      last_run = int(time.time())
+   fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
+   fd.close()
+   sys.exit(0)
+
+
+if __name__ == "__main__":
+   if 'UD_PROFILE' in os.environ:
+      import cProfile
+      import pstats
+      cProfile.run('ud_generate()', "udg_prof")
+      p = pstats.Stats('udg_prof')
+      ##p.sort_stats('time').print_stats()
+      p.sort_stats('cumulative').print_stats()

    else:
    else:

-     for k in Keyrings:
-       try: posix.remove(OutDir+os.path.basename(k));
-       except: pass;
+      ud_generate()

 
 # vim:set et:
 # vim:set ts=3:
 
 # vim:set et:
 # vim:set ts=3: