ud-mailgate: remove exception for münchen.debian.net
[mirror/userdir-ldap.git] / ud-generate
index cfd3f2a..a787dfa 100755 (executable)
@@ -28,6 +28,9 @@
 #   along with this program; if not, write to the Free Software
 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 
+from dsa_mq.connection import Connection
+from dsa_mq.config import Config
+
 import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
 from userdir_ldap import *
 from userdir_exceptions import *
@@ -71,8 +74,13 @@ isSSHFP = re.compile("^\s*IN\s+SSHFP")
 DNSZone = ".debian.net"
 Keyrings = ConfModule.sync_keyrings.split(":")
 GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
+GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
 GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
 MX_remap = json.loads(ConfModule.MX_remap)
+use_mq = getattr(ConfModule, "use_mq", True)
+
+rtc_realm = getattr(ConfModule, "rtc_realm", None)
+rtc_append = getattr(ConfModule, "rtc_append", None)
 
 def prettify(elem):
    """Return a pretty-printed XML string for the Element.
@@ -157,9 +165,6 @@ def IsRetired(account):
 
    return False
 
-#def IsGidDebian(account):
-#   return account['gidNumber'] == 800
-
 # See if this user is in the group list
 def IsInGroup(account, allowed, current_host):
   # See if the primary group is in the list
@@ -303,7 +308,7 @@ def GenShadowSudo(accounts, File, untrusted, current_host):
          Pass = '*'
          if 'sudoPassword' in a:
             for entry in a['sudoPassword']:
-               Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry)
+               Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
                if Match == None:
                   continue
                uuid = Match.group(1)
@@ -337,8 +342,10 @@ def GenShadowSudo(accounts, File, untrusted, current_host):
    Done(File, F, None)
 
 # Generate the sudo passwd file
-def GenSSHGitolite(accounts, hosts, File):
+def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
    F = None
+   if sshcommand is None:
+      sshcommand = GitoliteSSHCommand
    try:
       OldMask = os.umask(0022)
       F = open(File + ".tmp", "w", 0600)
@@ -349,19 +356,30 @@ def GenSSHGitolite(accounts, hosts, File):
             if not 'sshRSAAuthKey' in a: continue
 
             User = a['uid']
-            prefix = GitoliteSSHRestrictions.replace('@@USER@@', User)
+            prefix = GitoliteSSHRestrictions
+            prefix = prefix.replace('@@COMMAND@@', sshcommand)
+            prefix = prefix.replace('@@USER@@', User)
             for I in a["sshRSAAuthKey"]:
+               if I.startswith("allowed_hosts=") and ' ' in line:
+                  if current_host is None:
+                     continue
+                  machines, I = I.split('=', 1)[1].split(' ', 1)
+                  if current_host not in machines.split(','):
+                     continue # skip this key
+
                if I.startswith('ssh-'):
                   line = "%s %s"%(prefix, I)
                else:
-                  line = "%s,%s"%(prefix, I)
+                  continue # do not allow keys with other restrictions that might conflict
                line = Sanitize(line) + "\n"
                F.write(line)
 
          for dn, attrs in hosts:
             if not 'sshRSAHostKey' in attrs: continue
             hostname = "host-" + attrs['hostname'][0]
-            prefix = GitoliteSSHRestrictions.replace('@@USER@@', hostname)
+            prefix = GitoliteSSHRestrictions
+            prefix = prefix.replace('@@COMMAND@@', sshcommand)
+            prefix = prefix.replace('@@USER@@', hostname)
             for I in attrs["sshRSAHostKey"]:
                line = "%s %s"%(prefix, I)
                line = Sanitize(line) + "\n"
@@ -410,8 +428,8 @@ def GenWebPassword(accounts, File):
       Die(File, None, F)
       raise
 
-# Generate the voipPassword list
-def GenVoipPassword(accounts, File):
+# Generate the rtcPassword list
+def GenRtcPassword(accounts, File):
    F = None
    try:
       OldMask = os.umask(0077)
@@ -419,10 +437,11 @@ def GenVoipPassword(accounts, File):
       os.umask(OldMask)
 
       for a in accounts:
-         if not 'voipPassword' in a: continue
+         if a.is_guest_account(): continue
+         if not 'rtcPassword' in a: continue
          if not a.pw_active(): continue
 
-         Line = "%s@debian.org:%s:sip.debian.org:AUTHORIZED" % (a['uid'], str(a['voipPassword']))
+         Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
          Line = Sanitize(Line) + "\n"
          F.write("%s" % (Line))
 
@@ -430,6 +449,28 @@ def GenVoipPassword(accounts, File):
       Die(File, None, F)
       raise
 
+# Generate the TOTP auth file
+def GenTOTPSeed(accounts, File):
+   F = None
+   try:
+      OldMask = os.umask(0077)
+      F = open(File, "w", 0600)
+      os.umask(OldMask)
+
+      F.write("# Option User Prefix Seed\n")
+      for a in accounts:
+         if a.is_guest_account(): continue
+         if not 'totpSeed' in a: continue
+         if not a.pw_active(): continue
+
+         Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
+         Line = Sanitize(Line) + "\n"
+         F.write("%s" % (Line))
+   except:
+      Die(File, None, F)
+      raise
+
+
 def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
    OldMask = os.umask(0077)
    tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
@@ -847,6 +888,7 @@ def is_ipv6_addr(i):
    return True
 
 def ExtractDNSInfo(x):
+   hostname = GetAttr(x, "hostname")
 
    TTLprefix="\t"
    if 'dnsTTL' in x[1]:
@@ -856,38 +898,54 @@ def ExtractDNSInfo(x):
    if x[1].has_key("ipHostNumber"):
       for I in x[1]["ipHostNumber"]:
          if is_ipv6_addr(I):
-            DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I))
+            DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I))
          else:
-            DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I))
+            DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I))
 
    Algorithm = None
 
+   ssh_hostnames = [ hostname ]
+   if x[1].has_key("sshfpHostname"):
+      ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ]
+
    if 'sshRSAHostKey' in x[1]:
       for I in x[1]["sshRSAHostKey"]:
          Split = I.split()
-         if Split[0] == 'ssh-rsa':
+         key_prefix = Split[0]
+         key = base64.decodestring(Split[1])
+
+         # RFC4255
+         # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
+         if key_prefix == 'ssh-rsa':
             Algorithm = 1
-         if Split[0] == 'ssh-dss':
+         if key_prefix == 'ssh-dss':
             Algorithm = 2
+         if key_prefix == 'ssh-ed25519':
+            Algorithm = 4
          if Algorithm == None:
             continue
-         Fingerprint = hashlib.new('sha1', base64.decodestring(Split[1])).hexdigest()
-         DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint))
+         # and more from the registry
+         sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ]
+
+         fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ]
+         for h in ssh_hostnames:
+            for digest_codepoint, fingerprint in fingerprints:
+               DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))
 
    if 'architecture' in x[1]:
       Arch = GetAttr(x, "architecture")
       Mach = ""
       if x[1].has_key("machine"):
          Mach = " " + GetAttr(x, "machine")
-      DNSInfo.append("%sIN\tHINFO\t\"%s%s\" \"%s\"" % (TTLprefix, Arch, Mach, "Debian GNU/Linux"))
+      DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))
 
    if x[1].has_key("mXRecord"):
       for I in x[1]["mXRecord"]:
          if I in MX_remap:
             for e in MX_remap[I]:
-               DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, e))
+               DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))
          else:
-            DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I))
+            DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I))
 
    return DNSInfo
 
@@ -905,40 +963,9 @@ def GenZoneRecords(host_attrs, File):
          if IsDebianHost.match(GetAttr(x, "hostname")) is None:
             continue
 
-         DNSInfo = ExtractDNSInfo(x)
-         start = True
-         for Line in DNSInfo:
-            if start == True:
-               Line = "%s.\t%s" % (GetAttr(x, "hostname"), Line)
-               start = False
-            else:
-               Line = "\t\t\t%s" % (Line)
-
+         for Line in ExtractDNSInfo(x):
             F.write(Line + "\n")
 
-        # this would write sshfp lines for services on machines
-        # but we can't yet, since some are cnames and we'll make
-        # an invalid zonefile
-        #
-        # for i in x[1].get("purpose", []):
-        #    m = PurposeHostField.match(i)
-        #    if m:
-        #       m = m.group(1)
-        #       # we ignore [[*..]] entries
-        #       if m.startswith('*'):
-        #          continue
-        #       if m.startswith('-'):
-        #          m = m[1:]
-        #       if m:
-        #          if not m.endswith(HostDomain):
-        #             continue
-        #          if not m.endswith('.'):
-        #             m = m + "."
-        #          for Line in DNSInfo:
-        #             if isSSHFP.match(Line):
-        #                Line = "%s\t%s" % (m, Line)
-        #                F.write(Line + "\n")
-
    # Oops, something unspeakable happened.
    except:
       Die(File, F, None)
@@ -1102,7 +1129,8 @@ def get_accounts(ldap_conn):
                     "keyFingerPrint", "privateSub", "mailDisableMessage",\
                     "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
                     "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
-                    "mailContentInspectionAction", "webPassword", "voipPassword"])
+                    "mailContentInspectionAction", "webPassword", "rtcPassword",\
+                    "bATVToken", "totpSeed"])
 
    if passwd_attrs is None:
       raise UDEmptyList, "No Users"
@@ -1115,7 +1143,8 @@ def get_hosts(ldap_conn):
    # Fetch all the hosts
    HostAttrs    = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
                    ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
-                    "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"])
+                    "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
+                    "sshfpHostname"])
 
    if HostAttrs == None:
       raise UDEmptyList, "No Hosts"
@@ -1173,7 +1202,6 @@ def generate_all(global_dir, ldap_conn):
    accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
 
    accounts = filter(lambda x: not IsRetired(x), accounts)
-   #accounts_DDs = filter(lambda x: IsGidDebian(x), accounts)
 
    CheckForward(accounts)
 
@@ -1190,7 +1218,8 @@ def generate_all(global_dir, ldap_conn):
    GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
    GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
    GenWebPassword(accounts, global_dir + "web-passwords")
-   GenVoipPassword(accounts, global_dir + "voip-passwords")
+   GenRtcPassword(accounts, global_dir + "rtc-passwords")
+   GenTOTPSeed(accounts, global_dir + "users.oath")
    GenKeyrings(global_dir)
 
    # Compatibility.
@@ -1203,7 +1232,6 @@ def generate_all(global_dir, ldap_conn):
    GenMarkers(accounts, global_dir + "markers")
    GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
    GenHosts(host_attrs, global_dir + "debianhosts")
-   GenSSHGitolite(accounts, host_attrs, global_dir + "ssh-gitolite")
 
    GenDNS(accounts, global_dir + "dns-zone")
    GenZoneRecords(host_attrs, global_dir + "dns-sshfp")
@@ -1298,20 +1326,32 @@ def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
       DoLink(global_dir, OutDir, "debian-private")
 
    if 'GITOLITE' in ExtraList:
-      DoLink(global_dir, OutDir, "ssh-gitolite")
+      GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)
    if 'exportOptions' in host[1]:
       for entry in host[1]['exportOptions']:
          v = entry.split('=',1)
          if v[0] != 'GITOLITE' or len(v) != 2: continue
-         gitolite_accounts = filter(lambda x: IsInGroup(x, [v[1]], current_host), all_accounts)
-         gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
-         GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(v[1],))
+         options = v[1].split(',')
+         group = options.pop(0);
+         gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
+         if not 'nohosts' in options:
+            gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
+         else:
+            gitolite_hosts = []
+         command = None
+         for opt in options:
+            if opt.startswith('sshcmd='):
+               command = opt.split('=',1)[1]
+         GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)
 
    if 'WEB-PASSWORDS' in ExtraList:
       DoLink(global_dir, OutDir, "web-passwords")
 
-   if 'VOIP-PASSWORDS' in ExtraList:
-      DoLink(global_dir, OutDir, "voip-passwords")
+   if 'RTC-PASSWORDS' in ExtraList:
+      DoLink(global_dir, OutDir, "rtc-passwords")
+
+   if 'TOTP' in ExtraList:
+      DoLink(global_dir, OutDir, "users.oath")
 
    if 'KEYRING' in ExtraList:
       for k in Keyrings:
@@ -1382,6 +1422,34 @@ def getLastBuildTime(gdir):
 
    return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
 
+def mq_notify(options, message):
+   options.section = 'dsa-udgenerate'
+   options.config = '/etc/dsa/pubsub.conf'
+
+   config = Config(options)
+   conf = {
+      'rabbit_userid': config.username,
+      'rabbit_password': config.password,
+      'rabbit_virtual_host': config.vhost,
+      'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
+      'use_ssl': False
+   }
+
+   msg = {
+      'message': message,
+      'timestamp': int(time.time())
+   }
+   conn = None
+   try:
+      conn = Connection(conf=conf)
+      conn.topic_send(config.topic,
+            json.dumps(msg),
+            exchange_name=config.exchange,
+            timeout=5)
+   finally:
+      if conn:
+         conn.close()
+
 def ud_generate():
    parser = optparse.OptionParser()
    parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
@@ -1417,16 +1485,16 @@ def ud_generate():
 
    need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
 
-   if not options.force and not need_update:
-      fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
-      fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
-      fd.close()
-      sys.exit(0)
-
-   tracefd = open(os.path.join(generate_dir, "last_update.trace"), "w")
-   generate_all(generate_dir, l)
-   tracefd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, time_started))
-   tracefd.close()
+   fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
+   if need_update or options.force:
+      msg = 'Update forced' if options.force else 'Update needed'
+      generate_all(generate_dir, l)
+      if use_mq:
+         mq_notify(options, msg)
+      last_run = int(time.time())
+   fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
+   fd.close()
+   sys.exit(0)
 
 
 if __name__ == "__main__":