ud-generate: deal with users without loginShell

[mirror/userdir-ldap.git] / ud-generate

diff --git a/ud-generate b/ud-generate
index 950fd95..db6770d 100755 (executable)
--- a/ud-generate
+++ b/ud-generate
@@ -28,7 +28,11 @@
 #   along with this program; if not, write to the Free Software
 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 
 #   along with this program; if not, write to the Free Software
 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 

+from dsa_mq.connection import Connection
+from dsa_mq.config import Config
+

 import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
 import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm

+import subprocess

 from userdir_ldap import *
 from userdir_exceptions import *
 import UDLdap
 from userdir_ldap import *
 from userdir_exceptions import *
 import UDLdap


@@ -43,7 +47,7 @@ try:
    import simplejson as json
 except ImportError:
    import json
    import simplejson as json
 except ImportError:
    import json

-   if not '__author__' in json.__dict__:
+   if '__author__' not in json.__dict__:

       sys.stderr.write("Warning: This is probably the wrong json module.  We want python 2.6's json\n")
       sys.stderr.write("module, or simplejson on pytyon 2.5.  Let's see if/how stuff blows up.\n")
 
       sys.stderr.write("Warning: This is probably the wrong json module.  We want python 2.6's json\n")
       sys.stderr.write("module, or simplejson on pytyon 2.5.  Let's see if/how stuff blows up.\n")
 


@@ -71,8 +75,13 @@ isSSHFP = re.compile("^\s*IN\s+SSHFP")
 DNSZone = ".debian.net"
 Keyrings = ConfModule.sync_keyrings.split(":")
 GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
 DNSZone = ".debian.net"
 Keyrings = ConfModule.sync_keyrings.split(":")
 GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)

+GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)

 GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
 MX_remap = json.loads(ConfModule.MX_remap)
 GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
 MX_remap = json.loads(ConfModule.MX_remap)

+use_mq = getattr(ConfModule, "use_mq", True)
+
+rtc_realm = getattr(ConfModule, "rtc_realm", None)
+rtc_append = getattr(ConfModule, "rtc_append", None)

 
 def prettify(elem):
    """Return a pretty-printed XML string for the Element.
 
 def prettify(elem):
    """Return a pretty-printed XML string for the Element.


@@ -157,9 +166,6 @@ def IsRetired(account):
 
    return False
 
 
    return False
 

-#def IsGidDebian(account):
-#   return account['gidNumber'] == 800
-

 # See if this user is in the group list
 def IsInGroup(account, allowed, current_host):
   # See if the primary group is in the list
 # See if this user is in the group list
 def IsInGroup(account, allowed, current_host):
   # See if the primary group is in the list


@@ -169,7 +175,7 @@ def IsInGroup(account, allowed, current_host):
   if account.is_allowed_by_hostacl(current_host): return True
 
   # See if there are supplementary groups
   if account.is_allowed_by_hostacl(current_host): return True
 
   # See if there are supplementary groups

-  if not 'supplementaryGid' in account: return False
+  if 'supplementaryGid' not in account: return False

 
   supgroups=[]
   addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
 
   supgroups=[]
   addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)


@@ -179,9 +185,9 @@ def IsInGroup(account, allowed, current_host):
   return False
 
 def Die(File, F, Fdb):
   return False
 
 def Die(File, F, Fdb):

-   if F != None:
+   if F is not None:

       F.close()
       F.close()

-   if Fdb != None:
+   if Fdb is not None:

       Fdb.close()
    try: 
       os.remove(File + ".tmp")
       Fdb.close()
    try: 
       os.remove(File + ".tmp")


@@ -193,10 +199,10 @@ def Die(File, F, Fdb):
       pass
 
 def Done(File, F, Fdb):
       pass
 
 def Done(File, F, Fdb):

-   if F != None:
+   if F is not None:

       F.close()
       os.rename(File + ".tmp", File)
       F.close()
       os.rename(File + ".tmp", File)

-   if Fdb != None:
+   if Fdb is not None:

       Fdb.close()
       os.rename(File + ".tdb.tmp", File + ".tdb")
 
       Fdb.close()
       os.rename(File + ".tdb.tmp", File + ".tdb")
 


@@ -209,6 +215,8 @@ def GenPasswd(accounts, File, HomePrefix, PwdMarker):
       userlist = {}
       i = 0
       for a in accounts:
       userlist = {}
       i = 0
       for a in accounts:

+         if 'loginShell' not in a:
+             continue

          # Do not let people try to buffer overflow some busted passwd parser.
          if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue
 
          # Do not let people try to buffer overflow some busted passwd parser.
          if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue
 


@@ -303,8 +311,8 @@ def GenShadowSudo(accounts, File, untrusted, current_host):
          Pass = '*'
          if 'sudoPassword' in a:
             for entry in a['sudoPassword']:
          Pass = '*'
          if 'sudoPassword' in a:
             for entry in a['sudoPassword']:

-               Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry)
-               if Match == None:
+               Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
+               if Match is None:

                   continue
                uuid = Match.group(1)
                status = Match.group(2)
                   continue
                uuid = Match.group(1)
                status = Match.group(2)


@@ -337,8 +345,10 @@ def GenShadowSudo(accounts, File, untrusted, current_host):
    Done(File, F, None)
 
 # Generate the sudo passwd file
    Done(File, F, None)
 
 # Generate the sudo passwd file

-def GenSSHGitolite(accounts, hosts, File):
+def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):

    F = None
    F = None

+   if sshcommand is None:
+      sshcommand = GitoliteSSHCommand

    try:
       OldMask = os.umask(0022)
       F = open(File + ".tmp", "w", 0600)
    try:
       OldMask = os.umask(0022)
       F = open(File + ".tmp", "w", 0600)


@@ -346,22 +356,33 @@ def GenSSHGitolite(accounts, hosts, File):
 
       if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
          for a in accounts:
 
       if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
          for a in accounts:

-            if not 'sshRSAAuthKey' in a: continue
+            if 'sshRSAAuthKey' not in a: continue

 
             User = a['uid']
 
             User = a['uid']

-            prefix = GitoliteSSHRestrictions.replace('@@USER@@', User)
+            prefix = GitoliteSSHRestrictions
+            prefix = prefix.replace('@@COMMAND@@', sshcommand)
+            prefix = prefix.replace('@@USER@@', User)

             for I in a["sshRSAAuthKey"]:
             for I in a["sshRSAAuthKey"]:

+               if I.startswith("allowed_hosts=") and ' ' in line:
+                  if current_host is None:
+                     continue
+                  machines, I = I.split('=', 1)[1].split(' ', 1)
+                  if current_host not in machines.split(','):
+                     continue # skip this key
+

                if I.startswith('ssh-'):
                   line = "%s %s"%(prefix, I)
                else:
                if I.startswith('ssh-'):
                   line = "%s %s"%(prefix, I)
                else:

-                  line = "%s,%s"%(prefix, I)
+                  continue # do not allow keys with other restrictions that might conflict

                line = Sanitize(line) + "\n"
                F.write(line)
 
          for dn, attrs in hosts:
                line = Sanitize(line) + "\n"
                F.write(line)
 
          for dn, attrs in hosts:

-            if not 'sshRSAHostKey' in attrs: continue
+            if 'sshRSAHostKey' not in attrs: continue

             hostname = "host-" + attrs['hostname'][0]
             hostname = "host-" + attrs['hostname'][0]

-            prefix = GitoliteSSHRestrictions.replace('@@USER@@', hostname)
+            prefix = GitoliteSSHRestrictions
+            prefix = prefix.replace('@@COMMAND@@', sshcommand)
+            prefix = prefix.replace('@@USER@@', hostname)

             for I in attrs["sshRSAHostKey"]:
                line = "%s %s"%(prefix, I)
                line = Sanitize(line) + "\n"
             for I in attrs["sshRSAHostKey"]:
                line = "%s %s"%(prefix, I)
                line = Sanitize(line) + "\n"


@@ -379,7 +400,7 @@ def GenSSHShadow(global_dir, accounts):
    userkeys = {}
 
    for a in accounts:
    userkeys = {}
 
    for a in accounts:

-      if not 'sshRSAAuthKey' in a: continue
+      if 'sshRSAAuthKey' not in a: continue

 
       contents = []
       for I in a['sshRSAAuthKey']:
 
       contents = []
       for I in a['sshRSAAuthKey']:


@@ -398,7 +419,7 @@ def GenWebPassword(accounts, File):
       os.umask(OldMask)
 
       for a in accounts:
       os.umask(OldMask)
 
       for a in accounts:

-         if not 'webPassword' in a: continue
+         if 'webPassword' not in a: continue

          if not a.pw_active(): continue
 
          Pass = str(a['webPassword'])
          if not a.pw_active(): continue
 
          Pass = str(a['webPassword'])


@@ -410,44 +431,49 @@ def GenWebPassword(accounts, File):
       Die(File, None, F)
       raise
 
       Die(File, None, F)
       raise
 

-# Generate the voipPassword list
-def GenVoipPassword(accounts, File):
+# Generate the rtcPassword list
+def GenRtcPassword(accounts, File):

    F = None
    try:
       OldMask = os.umask(0077)
       F = open(File, "w", 0600)
       os.umask(OldMask)
 
    F = None
    try:
       OldMask = os.umask(0077)
       F = open(File, "w", 0600)
       os.umask(OldMask)
 

-      root = Element('include')
-

       for a in accounts:
       for a in accounts:

-         if not 'voipPassword' in a: continue
+         if a.is_guest_account(): continue
+         if 'rtcPassword' not in a: continue

          if not a.pw_active(): continue
 
          if not a.pw_active(): continue
 

-         Pass = str(a['voipPassword'])
-         user = Element('user')
-         user.attrib['id'] = "%s" % (a['uid'])
-         root.append(user)
-         params = Element('params')
-         user.append(params)
-         param = Element('param')
-         params.append(param)
-         param.attrib['name'] = "a1-hash"
-         param.attrib['value'] = "%s" % (Pass)
-         variables = Element('variables')
-         user.append(variables)
-         variable = Element('variable')
-         variable.attrib['name'] = "toll_allow"
-         variable.attrib['value'] = "domestic,international,local"
-         variables.append(variable)
-
-      F.write("%s" % (prettify(root)))
+         Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
+         Line = Sanitize(Line) + "\n"
+         F.write("%s" % (Line))

 
 

+   except:
+      Die(File, None, F)
+      raise

 
 

+# Generate the TOTP auth file
+def GenTOTPSeed(accounts, File):
+   F = None
+   try:
+      OldMask = os.umask(0077)
+      F = open(File, "w", 0600)
+      os.umask(OldMask)
+
+      F.write("# Option User Prefix Seed\n")
+      for a in accounts:
+         if a.is_guest_account(): continue
+         if 'totpSeed' not in a: continue
+         if not a.pw_active(): continue
+
+         Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
+         Line = Sanitize(Line) + "\n"
+         F.write("%s" % (Line))

    except:
       Die(File, None, F)
       raise
 
    except:
       Die(File, None, F)
       raise
 

+

 def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
    OldMask = os.umask(0077)
    tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
 def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
    OldMask = os.umask(0077)
    tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')


@@ -550,7 +576,7 @@ def GenGroup(accounts, File, current_host):
       # Sort them into a list of groups having a set of users
       for a in accounts:
          GroupHasPrimaryMembers[ a['gidNumber'] ] = True
       # Sort them into a list of groups having a set of users
       for a in accounts:
          GroupHasPrimaryMembers[ a['gidNumber'] ] = True

-         if not 'supplementaryGid' in a: continue
+         if 'supplementaryGid' not in a: continue

 
          supgroups=[]
          addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
 
          supgroups=[]
          addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)


@@ -560,7 +586,7 @@ def GenGroup(accounts, File, current_host):
       # Output the group file.
       J = 0
       for x in GroupMap.keys():
       # Output the group file.
       J = 0
       for x in GroupMap.keys():

-         if not x in GroupIDMap:
+         if x not in GroupIDMap:

             continue
 
          if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
             continue
 
          if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:


@@ -589,7 +615,7 @@ def GenGroup(accounts, File, current_host):
 
 def CheckForward(accounts):
    for a in accounts:
 
 def CheckForward(accounts):
    for a in accounts:

-      if not 'emailForward' in a: continue
+      if 'emailForward' not in a: continue

 
       delete = False
 
 
       delete = False
 


@@ -610,7 +636,7 @@ def GenForward(accounts, File):
       os.umask(OldMask)
 
       for a in accounts:
       os.umask(OldMask)
 
       for a in accounts:

-         if not 'emailForward' in a: continue
+         if 'emailForward' not in a: continue

          Line = "%s: %s" % (a['uid'], a['emailForward'])
          Line = Sanitize(Line) + "\n"
          F.write(Line)
          Line = "%s: %s" % (a['uid'], a['emailForward'])
          Line = Sanitize(Line) + "\n"
          F.write(Line)


@@ -622,28 +648,24 @@ def GenForward(accounts, File):
    Done(File, F, None)
 
 def GenCDB(accounts, File, key):
    Done(File, F, None)
 
 def GenCDB(accounts, File, key):

-   Fdb = None
+   prefix = ["/usr/bin/eatmydata"] if os.path.exists('/usr/bin/eatmydata') else []
+   # nothing else does the fsync stuff, so why do it here?
+   Fdb = subprocess.Popen(prefix + ["cdbmake", File, "%s.tmp" % File],
+                          preexec_fn=lambda: os.umask(0022),
+                          stdin=subprocess.PIPE)

    try:
    try:

-      OldMask = os.umask(0022)
-      # nothing else does the fsync stuff, so why do it here?
-      prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else ''
-      Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w")
-      os.umask(OldMask)
-

       # Write out the email address for each user
       for a in accounts:
       # Write out the email address for each user
       for a in accounts:

-         if not key in a: continue
+         if key not in a: continue

          value = a[key]
          user = a['uid']
          value = a[key]
          user = a['uid']

-         Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
+         Fdb.stdin.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))

 
 

-      Fdb.write("\n")
-   # Oops, something unspeakable happened.
-   except:
-      Fdb.close()
-      raise
-   if Fdb.close() != None:
-      raise "cdbmake gave an error"
+      Fdb.stdin.write("\n")
+   finally:
+      Fdb.stdin.close()
+      if Fdb.wait() != 0:
+         raise Exception("cdbmake gave an error")

 
 def GenDBM(accounts, File, key):
    Fdb = None
 
 def GenDBM(accounts, File, key):
    Fdb = None


@@ -655,21 +677,23 @@ def GenDBM(accounts, File, key):
       pass
 
    try:
       pass
 
    try:

-      Fdb = dbm.open(fn + ".tmp", "c")
+      Fdb = dbm.open(fn, "c")

       os.umask(OldMask)
 
       # Write out the email address for each user
       for a in accounts:
       os.umask(OldMask)
 
       # Write out the email address for each user
       for a in accounts:

-         if not key in a: continue
+         if key not in a: continue

          value = a[key]
          user = a['uid']
          Fdb[user] = value
 
       Fdb.close()
    except:
          value = a[key]
          user = a['uid']
          Fdb[user] = value
 
       Fdb.close()
    except:

-      Die(File, F, None)
+      # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
+      os.remove(File + ".db")

       raise
       raise

-   Done(File, F, None)
+   # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
+   os.rename (File + ".db", File)

 
 # Generate the anon XEarth marker file
 def GenMarkers(accounts, File):
 
 # Generate the anon XEarth marker file
 def GenMarkers(accounts, File):


@@ -703,7 +727,7 @@ def GenPrivate(accounts, File):
       for a in accounts:
          if not a.is_active_user(): continue
          if a.is_guest_account(): continue
       for a in accounts:
          if not a.is_active_user(): continue
          if a.is_guest_account(): continue

-         if not 'privateSub' in a: continue
+         if 'privateSub' not in a: continue

          try:
             Line = "%s"%(a['privateSub'])
             Line = Sanitize(Line) + "\n"
          try:
             Line = "%s"%(a['privateSub'])
             Line = Sanitize(Line) + "\n"


@@ -745,7 +769,7 @@ def GenMailDisable(accounts, File):
       F = open(File + ".tmp", "w")
 
       for a in accounts:
       F = open(File + ".tmp", "w")
 
       for a in accounts:

-         if not 'mailDisableMessage' in a: continue
+         if 'mailDisableMessage' not in a: continue

          Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
          Line = Sanitize(Line) + "\n"
          F.write(Line)
          Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
          Line = Sanitize(Line) + "\n"
          F.write(Line)


@@ -763,7 +787,7 @@ def GenMailBool(accounts, File, key):
       F = open(File + ".tmp", "w")
 
       for a in accounts:
       F = open(File + ".tmp", "w")
 
       for a in accounts:

-         if not key in a: continue
+         if key not in a: continue

          if not a[key] == 'TRUE': continue
          Line = "%s"%(a['uid'])
          Line = Sanitize(Line) + "\n"
          if not a[key] == 'TRUE': continue
          Line = "%s"%(a['uid'])
          Line = Sanitize(Line) + "\n"


@@ -785,7 +809,7 @@ def GenMailList(accounts, File, key):
       else:                      validregex = re.compile('^[-\w.]+$')
 
       for a in accounts:
       else:                      validregex = re.compile('^[-\w.]+$')
 
       for a in accounts:

-         if not key in a: continue
+         if key not in a: continue

 
          filtered = filter(lambda z: validregex.match(z), a[key])
          if len(filtered) == 0: continue
 
          filtered = filter(lambda z: validregex.match(z), a[key])
          if len(filtered) == 0: continue


@@ -814,7 +838,7 @@ def GenDNS(accounts, File):
 
       # Write out the zone file entry for each user
       for a in accounts:
 
       # Write out the zone file entry for each user
       for a in accounts:

-         if not 'dnsZoneEntry' in a: continue
+         if 'dnsZoneEntry' not in a: continue

          if not a.is_active_user() and not isRoleAccount(a): continue
          if a.is_guest_account(): continue
 
          if not a.is_active_user() and not isRoleAccount(a): continue
          if a.is_guest_account(): continue
 


@@ -827,7 +851,7 @@ def GenDNS(accounts, File):
                   F.write(Line)
 
                   Host = Split[0] + DNSZone
                   F.write(Line)
 
                   Host = Split[0] + DNSZone

-                  if BSMTPCheck.match(Line) != None:
+                  if BSMTPCheck.match(Line) is not None:

                      F.write("; Has BSMTP\n")
 
                   # Write some identification information
                      F.write("; Has BSMTP\n")
 
                   # Write some identification information


@@ -863,6 +887,7 @@ def is_ipv6_addr(i):
    return True
 
 def ExtractDNSInfo(x):
    return True
 
 def ExtractDNSInfo(x):

+   hostname = GetAttr(x, "hostname")

 
    TTLprefix="\t"
    if 'dnsTTL' in x[1]:
 
    TTLprefix="\t"
    if 'dnsTTL' in x[1]:


@@ -872,38 +897,54 @@ def ExtractDNSInfo(x):
    if x[1].has_key("ipHostNumber"):
       for I in x[1]["ipHostNumber"]:
          if is_ipv6_addr(I):
    if x[1].has_key("ipHostNumber"):
       for I in x[1]["ipHostNumber"]:
          if is_ipv6_addr(I):

-            DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I))
+            DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I))

          else:
          else:

-            DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I))
+            DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I))

 
    Algorithm = None
 
 
    Algorithm = None
 

+   ssh_hostnames = [ hostname ]
+   if x[1].has_key("sshfpHostname"):
+      ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ]
+

    if 'sshRSAHostKey' in x[1]:
       for I in x[1]["sshRSAHostKey"]:
          Split = I.split()
    if 'sshRSAHostKey' in x[1]:
       for I in x[1]["sshRSAHostKey"]:
          Split = I.split()

-         if Split[0] == 'ssh-rsa':
+         key_prefix = Split[0]
+         key = base64.decodestring(Split[1])
+
+         # RFC4255
+         # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
+         if key_prefix == 'ssh-rsa':

             Algorithm = 1
             Algorithm = 1

-         if Split[0] == 'ssh-dss':
+         if key_prefix == 'ssh-dss':

             Algorithm = 2
             Algorithm = 2

-         if Algorithm == None:
+         if key_prefix == 'ssh-ed25519':
+            Algorithm = 4
+         if Algorithm is None:

             continue
             continue

-         Fingerprint = hashlib.new('sha1', base64.decodestring(Split[1])).hexdigest()
-         DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint))
+         # and more from the registry
+         sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ]
+
+         fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ]
+         for h in ssh_hostnames:
+            for digest_codepoint, fingerprint in fingerprints:
+               DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))

 
    if 'architecture' in x[1]:
       Arch = GetAttr(x, "architecture")
       Mach = ""
       if x[1].has_key("machine"):
          Mach = " " + GetAttr(x, "machine")
 
    if 'architecture' in x[1]:
       Arch = GetAttr(x, "architecture")
       Mach = ""
       if x[1].has_key("machine"):
          Mach = " " + GetAttr(x, "machine")

-      DNSInfo.append("%sIN\tHINFO\t\"%s%s\" \"%s\"" % (TTLprefix, Arch, Mach, "Debian GNU/Linux"))
+      DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))

 
    if x[1].has_key("mXRecord"):
       for I in x[1]["mXRecord"]:
          if I in MX_remap:
             for e in MX_remap[I]:
 
    if x[1].has_key("mXRecord"):
       for I in x[1]["mXRecord"]:
          if I in MX_remap:
             for e in MX_remap[I]:

-               DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, e))
+               DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))

          else:
          else:

-            DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I))
+            DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I))

 
    return DNSInfo
 
 
    return DNSInfo
 


@@ -921,40 +962,9 @@ def GenZoneRecords(host_attrs, File):
          if IsDebianHost.match(GetAttr(x, "hostname")) is None:
             continue
 
          if IsDebianHost.match(GetAttr(x, "hostname")) is None:
             continue
 

-         DNSInfo = ExtractDNSInfo(x)
-         start = True
-         for Line in DNSInfo:
-            if start == True:
-               Line = "%s.\t%s" % (GetAttr(x, "hostname"), Line)
-               start = False
-            else:
-               Line = "\t\t\t%s" % (Line)
-
+         for Line in ExtractDNSInfo(x):

             F.write(Line + "\n")
 
             F.write(Line + "\n")
 

-        # this would write sshfp lines for services on machines
-        # but we can't yet, since some are cnames and we'll make
-        # an invalid zonefile
-        #
-        # for i in x[1].get("purpose", []):
-        #    m = PurposeHostField.match(i)
-        #    if m:
-        #       m = m.group(1)
-        #       # we ignore [[*..]] entries
-        #       if m.startswith('*'):
-        #          continue
-        #       if m.startswith('-'):
-        #          m = m[1:]
-        #       if m:
-        #          if not m.endswith(HostDomain):
-        #             continue
-        #          if not m.endswith('.'):
-        #             m = m + "."
-        #          for Line in DNSInfo:
-        #             if isSSHFP.match(Line):
-        #                Line = "%s\t%s" % (m, Line)
-        #                F.write(Line + "\n")
-

    # Oops, something unspeakable happened.
    except:
       Die(File, F, None)
    # Oops, something unspeakable happened.
    except:
       Die(File, F, None)


@@ -969,7 +979,7 @@ def GenBSMTP(accounts, File, HomePrefix):
      
       # Write out the zone file entry for each user
       for a in accounts:
      
       # Write out the zone file entry for each user
       for a in accounts:

-         if not 'dnsZoneEntry' in a: continue
+         if 'dnsZoneEntry' not in a: continue

          if not a.is_active_user(): continue
 
          try:
          if not a.is_active_user(): continue
 
          try:


@@ -982,7 +992,7 @@ def GenBSMTP(accounts, File, HomePrefix):
                   Line = " ".join(Split) + "\n"
      
                   Host = Split[0] + DNSZone
                   Line = " ".join(Split) + "\n"
      
                   Host = Split[0] + DNSZone

-                  if BSMTPCheck.match(Line) != None:
+                  if BSMTPCheck.match(Line) is not None:

                       F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
                                   a['uid'], HomePrefix, a['uid'], Host))
      
                       F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
                                   a['uid'], HomePrefix, a['uid'], Host))
      


@@ -1077,7 +1087,7 @@ def GenHosts(host_attrs, File):
          if IsDebianHost.match(GetAttr(x, "hostname")) is None:
             continue
 
          if IsDebianHost.match(GetAttr(x, "hostname")) is None:
             continue
 

-         if not 'ipHostNumber' in x[1]:
+         if 'ipHostNumber' not in x[1]:

             continue
 
          addrs = x[1]["ipHostNumber"]
             continue
 
          addrs = x[1]["ipHostNumber"]


@@ -1118,7 +1128,8 @@ def get_accounts(ldap_conn):
                     "keyFingerPrint", "privateSub", "mailDisableMessage",\
                     "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
                     "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
                     "keyFingerPrint", "privateSub", "mailDisableMessage",\
                     "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
                     "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\

-                    "mailContentInspectionAction", "webPassword", "voipPassword"])
+                    "mailContentInspectionAction", "webPassword", "rtcPassword",\
+                    "bATVToken", "totpSeed", "mailDefaultOptions"])

 
    if passwd_attrs is None:
       raise UDEmptyList, "No Users"
 
    if passwd_attrs is None:
       raise UDEmptyList, "No Users"


@@ -1131,9 +1142,10 @@ def get_hosts(ldap_conn):
    # Fetch all the hosts
    HostAttrs    = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
                    ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
    # Fetch all the hosts
    HostAttrs    = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
                    ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\

-                    "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"])
+                    "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
+                    "sshfpHostname"])

 
 

-   if HostAttrs == None:
+   if HostAttrs is None:

       raise UDEmptyList, "No Hosts"
 
    HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))
       raise UDEmptyList, "No Hosts"
 
    HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))


@@ -1189,15 +1201,16 @@ def generate_all(global_dir, ldap_conn):
    accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
 
    accounts = filter(lambda x: not IsRetired(x), accounts)
    accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
 
    accounts = filter(lambda x: not IsRetired(x), accounts)

-   #accounts_DDs = filter(lambda x: IsGidDebian(x), accounts)

 
    CheckForward(accounts)
 
    GenMailDisable(accounts, global_dir + "mail-disable")
    GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
 
    CheckForward(accounts)
 
    GenMailDisable(accounts, global_dir + "mail-disable")
    GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')

-   GenDBM(accounts, global_dir + "mail-forward", 'emailForward')
+   GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')

    GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
    GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')

-   GenDBM(accounts, global_dir + "mail-contentinspectionaction", 'mailContentInspectionAction')
+   GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
+   GenCDB(accounts, global_dir + "default-mail-options.cdb", 'mailDefaultOptions')
+   GenDBM(accounts, global_dir + "default-mail-options.db", 'mailDefaultOptions')

    GenPrivate(accounts, global_dir + "debian-private")
    GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
    GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
    GenPrivate(accounts, global_dir + "debian-private")
    GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
    GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")


@@ -1206,20 +1219,20 @@ def generate_all(global_dir, ldap_conn):
    GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
    GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
    GenWebPassword(accounts, global_dir + "web-passwords")
    GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
    GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
    GenWebPassword(accounts, global_dir + "web-passwords")

-   GenVoipPassword(accounts, global_dir + "voip-passwords")
+   GenRtcPassword(accounts, global_dir + "rtc-passwords")
+   GenTOTPSeed(accounts, global_dir + "users.oath")

    GenKeyrings(global_dir)
 
    # Compatibility.
    GenForward(accounts, global_dir + "forward-alias")
 
    GenAllUsers(accounts, global_dir + 'all-accounts.json')
    GenKeyrings(global_dir)
 
    # Compatibility.
    GenForward(accounts, global_dir + "forward-alias")
 
    GenAllUsers(accounts, global_dir + 'all-accounts.json')

-   accounts = filter(lambda a: not a in accounts_disabled, accounts)
+   accounts = filter(lambda a: a not in accounts_disabled, accounts)

 
    ssh_userkeys = GenSSHShadow(global_dir, accounts)
    GenMarkers(accounts, global_dir + "markers")
    GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
    GenHosts(host_attrs, global_dir + "debianhosts")
 
    ssh_userkeys = GenSSHShadow(global_dir, accounts)
    GenMarkers(accounts, global_dir + "markers")
    GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
    GenHosts(host_attrs, global_dir + "debianhosts")

-   GenSSHGitolite(accounts, host_attrs, global_dir + "ssh-gitolite")

 
    GenDNS(accounts, global_dir + "dns-zone")
    GenZoneRecords(host_attrs, global_dir + "dns-sshfp")
 
    GenDNS(accounts, global_dir + "dns-zone")
    GenZoneRecords(host_attrs, global_dir + "dns-sshfp")


@@ -1227,7 +1240,7 @@ def generate_all(global_dir, ldap_conn):
    setup_group_maps(ldap_conn)
 
    for host in host_attrs:
    setup_group_maps(ldap_conn)
 
    for host in host_attrs:

-      if not "hostname" in host[1]:
+      if "hostname" not in host[1]:

          continue
       generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
 
          continue
       generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
 


@@ -1273,11 +1286,11 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
    # the relevant ssh keys
    GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)
 
    # the relevant ssh keys
    GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)
 

-   if not 'NOPASSWD' in ExtraList:
+   if 'NOPASSWD' not in ExtraList:

       GenShadow(accounts, OutDir + "shadow")
 
    # Link in global things
       GenShadow(accounts, OutDir + "shadow")
 
    # Link in global things

-   if not 'NOMARKERS' in ExtraList:
+   if 'NOMARKERS' not in ExtraList:

       DoLink(global_dir, OutDir, "markers")
    DoLink(global_dir, OutDir, "mail-forward.cdb")
    DoLink(global_dir, OutDir, "mail-forward.db")
       DoLink(global_dir, OutDir, "markers")
    DoLink(global_dir, OutDir, "mail-forward.cdb")
    DoLink(global_dir, OutDir, "mail-forward.db")


@@ -1290,12 +1303,12 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
    DoLink(global_dir, OutDir, "mail-rhsbl")
    DoLink(global_dir, OutDir, "mail-whitelist")
    DoLink(global_dir, OutDir, "all-accounts.json")
    DoLink(global_dir, OutDir, "mail-rhsbl")
    DoLink(global_dir, OutDir, "mail-whitelist")
    DoLink(global_dir, OutDir, "all-accounts.json")

+   DoLink(global_dir, OutDir, "default-mail-options.cdb")
+   DoLink(global_dir, OutDir, "default-mail-options.db")

    GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
    GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')

-   GenDBM(accounts, OutDir + "user-forward", 'emailForward')
+   GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')

    GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
    GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')

-   GenDBM(accounts, OutDir + "batv-tokens", 'bATVToken')
-   GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions')
-   GenDBM(accounts, OutDir + "default-mail-options", 'mailDefaultOptions')
+   GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')

 
    # Compatibility.
    DoLink(global_dir, OutDir, "forward-alias")
 
    # Compatibility.
    DoLink(global_dir, OutDir, "forward-alias")


@@ -1314,20 +1327,32 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
       DoLink(global_dir, OutDir, "debian-private")
 
    if 'GITOLITE' in ExtraList:
       DoLink(global_dir, OutDir, "debian-private")
 
    if 'GITOLITE' in ExtraList:

-      DoLink(global_dir, OutDir, "ssh-gitolite")
+      GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)

    if 'exportOptions' in host[1]:
       for entry in host[1]['exportOptions']:
          v = entry.split('=',1)
          if v[0] != 'GITOLITE' or len(v) != 2: continue
    if 'exportOptions' in host[1]:
       for entry in host[1]['exportOptions']:
          v = entry.split('=',1)
          if v[0] != 'GITOLITE' or len(v) != 2: continue

-         gitolite_accounts = filter(lambda x: IsInGroup(x, [v[1]], current_host), all_accounts)
-         gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
-         GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(v[1],))
+         options = v[1].split(',')
+         group = options.pop(0)
+         gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
+         if 'nohosts' not in options:
+            gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
+         else:
+            gitolite_hosts = []
+         command = None
+         for opt in options:
+            if opt.startswith('sshcmd='):
+               command = opt.split('=',1)[1]
+         GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)

 
    if 'WEB-PASSWORDS' in ExtraList:
       DoLink(global_dir, OutDir, "web-passwords")
 
 
    if 'WEB-PASSWORDS' in ExtraList:
       DoLink(global_dir, OutDir, "web-passwords")
 

-   if 'VOIP-PASSWORDS' in ExtraList:
-      DoLink(global_dir, OutDir, "voip-passwords")
+   if 'RTC-PASSWORDS' in ExtraList:
+      DoLink(global_dir, OutDir, "rtc-passwords")
+
+   if 'TOTP' in ExtraList:
+      DoLink(global_dir, OutDir, "users.oath")

 
    if 'KEYRING' in ExtraList:
       for k in Keyrings:
 
    if 'KEYRING' in ExtraList:
       for k in Keyrings:


@@ -1398,6 +1423,34 @@ def getLastBuildTime(gdir):
 
    return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
 
 
    return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
 

+def mq_notify(options, message):
+   options.section = 'dsa-udgenerate'
+   options.config = '/etc/dsa/pubsub.conf'
+
+   config = Config(options)
+   conf = {
+      'rabbit_userid': config.username,
+      'rabbit_password': config.password,
+      'rabbit_virtual_host': config.vhost,
+      'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
+      'use_ssl': False
+   }
+
+   msg = {
+      'message': message,
+      'timestamp': int(time.time())
+   }
+   conn = None
+   try:
+      conn = Connection(conf=conf)
+      conn.topic_send(config.topic,
+            json.dumps(msg),
+            exchange_name=config.exchange,
+            timeout=5)
+   finally:
+      if conn:
+         conn.close()
+

 def ud_generate():
    parser = optparse.OptionParser()
    parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
 def ud_generate():
    parser = optparse.OptionParser()
    parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",


@@ -1433,16 +1486,16 @@ def ud_generate():
 
    need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
 
 
    need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
 

-   if not options.force and not need_update:
-      fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
-      fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
-      fd.close()
-      sys.exit(0)
-
-   tracefd = open(os.path.join(generate_dir, "last_update.trace"), "w")
-   generate_all(generate_dir, l)
-   tracefd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, time_started))
-   tracefd.close()
+   fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
+   if need_update or options.force:
+      msg = 'Update forced' if options.force else 'Update needed'
+      generate_all(generate_dir, l)
+      if use_mq:
+         mq_notify(options, msg)
+      last_run = int(time.time())
+   fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
+   fd.close()
+   sys.exit(0)

 
 
 if __name__ == "__main__":
 
 
 if __name__ == "__main__":