3 # Generates passwd, shadow and group files from the ldap directory.
5 # Copyright (c) 2000-2001 Jason Gunthorpe <jgg@debian.org>
6 # Copyright (c) 2003-2004 James Troup <troup@debian.org>
7 # Copyright (c) 2004-2005,7 Joey Schulze <joey@infodrom.org>
8 # Copyright (c) 2001-2007 Ryan Murray <rmurray@debian.org>
9 # Copyright (c) 2008,2009,2010,2011 Peter Palfrader <peter@palfrader.org>
10 # Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
11 # Copyright (c) 2008 Mark Hymers <mhy@debian.org>
12 # Copyright (c) 2008 Luk Claes <luk@debian.org>
13 # Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>
14 # Copyright (c) 2009 Stephen Gran <steve@lobefin.net>
15 # Copyright (c) 2010 Helmut Grohne <helmut@subdivi.de>
17 # This program is free software; you can redistribute it and/or modify
18 # it under the terms of the GNU General Public License as published by
19 # the Free Software Foundation; either version 2 of the License, or
20 # (at your option) any later version.
22 # This program is distributed in the hope that it will be useful,
23 # but WITHOUT ANY WARRANTY; without even the implied warranty of
24 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 # GNU General Public License for more details.
27 # You should have received a copy of the GNU General Public License
28 # along with this program; if not, write to the Free Software
29 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
31 from dsa_mq.connection import Connection
32 from dsa_mq.config import Config
34 import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
35 from userdir_ldap import *
36 from userdir_exceptions import *
38 from xml.etree.ElementTree import Element, SubElement, Comment
39 from xml.etree import ElementTree
40 from xml.dom import minidom
42 from cStringIO import StringIO
44 from StringIO import StringIO
46 import simplejson as json
49 if not '__author__' in json.__dict__:
50 sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n")
51 sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n")
54 sys.stderr.write("You should probably not run ud-generate as root.\n")
66 UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
69 EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
70 BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL)
71 PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
72 IsDebianHost = re.compile(ConfModule.dns_hostmatch)
73 isSSHFP = re.compile("^\s*IN\s+SSHFP")
74 DNSZone = ".debian.net"
75 Keyrings = ConfModule.sync_keyrings.split(":")
76 GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
77 GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
78 GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
79 MX_remap = json.loads(ConfModule.MX_remap)
80 use_mq = getattr(ConfModule, "use_mq", True)
82 rtc_realm = getattr(ConfModule, "rtc_realm", None)
83 rtc_append = getattr(ConfModule, "rtc_append", None)
86 """Return a pretty-printed XML string for the Element.
88 rough_string = ElementTree.tostring(elem, 'utf-8')
89 reparsed = minidom.parseString(rough_string)
90 return reparsed.toprettyxml(indent=" ")
92 def safe_makedirs(dir):
96 if e.errno == errno.EEXIST:
101 def safe_rmtree(dir):
105 if e.errno == errno.ENOENT:
110 def get_lock(fn, wait=5*60):
113 ends = time.time() + wait
118 fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
122 if time.time() >= ends:
124 sl = min(sl*2, 10, ends - time.time())
130 return Str.translate(string.maketrans("\n\r\t", "$$$"))
132 def DoLink(From, To, File):
134 posix.remove(To + File)
137 posix.link(From + File, To + File)
139 def IsRetired(account):
141 Looks for accountStatus in the LDAP record and tries to
142 match it against one of the known retired statuses
145 status = account['accountStatus']
147 line = status.split()
150 if status == "inactive":
153 elif status == "memorial":
156 elif status == "retiring":
157 # We'll give them a few extra days over what we said
158 age = 6 * 31 * 24 * 60 * 60
160 return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age
168 # See if this user is in the group list
169 def IsInGroup(account, allowed, current_host):
170 # See if the primary group is in the list
171 if str(account['gidNumber']) in allowed: return True
173 # Check the host based ACL
174 if account.is_allowed_by_hostacl(current_host): return True
176 # See if there are supplementary groups
177 if not 'supplementaryGid' in account: return False
180 addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
186 def Die(File, F, Fdb):
192 os.remove(File + ".tmp")
196 os.remove(File + ".tdb.tmp")
200 def Done(File, F, Fdb):
203 os.rename(File + ".tmp", File)
206 os.rename(File + ".tdb.tmp", File + ".tdb")
208 # Generate the password list
209 def GenPasswd(accounts, File, HomePrefix, PwdMarker):
212 F = open(File + ".tdb.tmp", "w")
217 # Do not let people try to buffer overflow some busted passwd parser.
218 if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue
220 userlist[a['uid']] = a['gidNumber']
221 line = "%s:%s:%d:%d:%s:%s%s:%s" % (
227 HomePrefix, a['uid'],
229 line = Sanitize(line) + "\n"
230 F.write("0%u %s" % (i, line))
231 F.write(".%s %s" % (a['uid'], line))
232 F.write("=%d %s" % (a['uidNumber'], line))
235 # Oops, something unspeakable happened.
241 # Return the list of users so we know which keys to export
244 def GenAllUsers(accounts, file):
247 OldMask = os.umask(0022)
248 f = open(file + ".tmp", "w", 0644)
253 all.append( { 'uid': a['uid'],
254 'uidNumber': a['uidNumber'],
255 'active': a.pw_active() and a.shadow_active() } )
258 # Oops, something unspeakable happened.
264 # Generate the shadow list
265 def GenShadow(accounts, File):
268 OldMask = os.umask(0077)
269 F = open(File + ".tdb.tmp", "w", 0600)
274 # If the account is locked, mark it as such in shadow
275 # See Debian Bug #308229 for why we set it to 1 instead of 0
276 if not a.pw_active(): ShadowExpire = '1'
277 elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire'])
278 else: ShadowExpire = ''
281 values.append(a['uid'])
282 values.append(a.get_password())
283 for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive':
284 if key in a: values.append(a[key])
285 else: values.append('')
286 values.append(ShadowExpire)
287 line = ':'.join(values)+':'
288 line = Sanitize(line) + "\n"
289 F.write("0%u %s" % (i, line))
290 F.write(".%s %s" % (a['uid'], line))
293 # Oops, something unspeakable happened.
299 # Generate the sudo passwd file
300 def GenShadowSudo(accounts, File, untrusted, current_host):
303 OldMask = os.umask(0077)
304 F = open(File + ".tmp", "w", 0600)
309 if 'sudoPassword' in a:
310 for entry in a['sudoPassword']:
311 Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
314 uuid = Match.group(1)
315 status = Match.group(2)
316 hosts = Match.group(3)
317 cryptedpass = Match.group(4)
319 if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
321 for_all = hosts == "*"
322 for_this_host = current_host in hosts.split(',')
323 if not (for_all or for_this_host):
325 # ignore * passwords for untrusted hosts, but copy host specific passwords
326 if for_all and untrusted:
329 if for_this_host: # this makes sure we take a per-host entry over the for-all entry
334 Line = "%s:%s" % (a['uid'], Pass)
335 Line = Sanitize(Line) + "\n"
336 F.write("%s" % (Line))
338 # Oops, something unspeakable happened.
344 # Generate the sudo passwd file
345 def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
347 if sshcommand is None:
348 sshcommand = GitoliteSSHCommand
350 OldMask = os.umask(0022)
351 F = open(File + ".tmp", "w", 0600)
354 if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
356 if not 'sshRSAAuthKey' in a: continue
359 prefix = GitoliteSSHRestrictions
360 prefix = prefix.replace('@@COMMAND@@', sshcommand)
361 prefix = prefix.replace('@@USER@@', User)
362 for I in a["sshRSAAuthKey"]:
363 if I.startswith("allowed_hosts=") and ' ' in line:
364 if current_host is None:
366 machines, I = I.split('=', 1)[1].split(' ', 1)
367 if current_host not in machines.split(','):
368 continue # skip this key
370 if I.startswith('ssh-'):
371 line = "%s %s"%(prefix, I)
373 continue # do not allow keys with other restrictions that might conflict
374 line = Sanitize(line) + "\n"
377 for dn, attrs in hosts:
378 if not 'sshRSAHostKey' in attrs: continue
379 hostname = "host-" + attrs['hostname'][0]
380 prefix = GitoliteSSHRestrictions
381 prefix = prefix.replace('@@COMMAND@@', sshcommand)
382 prefix = prefix.replace('@@USER@@', hostname)
383 for I in attrs["sshRSAHostKey"]:
384 line = "%s %s"%(prefix, I)
385 line = Sanitize(line) + "\n"
388 # Oops, something unspeakable happened.
394 # Generate the shadow list
395 def GenSSHShadow(global_dir, accounts):
396 # Fetch all the users
400 if not 'sshRSAAuthKey' in a: continue
403 for I in a['sshRSAAuthKey']:
404 MultipleLine = "%s" % I
405 MultipleLine = Sanitize(MultipleLine)
406 contents.append(MultipleLine)
407 userkeys[a['uid']] = contents
410 # Generate the webPassword list
411 def GenWebPassword(accounts, File):
414 OldMask = os.umask(0077)
415 F = open(File, "w", 0600)
419 if not 'webPassword' in a: continue
420 if not a.pw_active(): continue
422 Pass = str(a['webPassword'])
423 Line = "%s:%s" % (a['uid'], Pass)
424 Line = Sanitize(Line) + "\n"
425 F.write("%s" % (Line))
431 # Generate the rtcPassword list
432 def GenRtcPassword(accounts, File):
435 OldMask = os.umask(0077)
436 F = open(File, "w", 0600)
440 if a.is_guest_account(): continue
441 if not 'rtcPassword' in a: continue
442 if not a.pw_active(): continue
444 Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
445 Line = Sanitize(Line) + "\n"
446 F.write("%s" % (Line))
452 # Generate the TOTP auth file
453 def GenTOTPSeed(accounts, File):
456 OldMask = os.umask(0077)
457 F = open(File, "w", 0600)
460 F.write("# Option User Prefix Seed\n")
462 if a.is_guest_account(): continue
463 if not 'totpSeed' in a: continue
464 if not a.pw_active(): continue
466 Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
467 Line = Sanitize(Line) + "\n"
468 F.write("%s" % (Line))
474 def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
475 OldMask = os.umask(0077)
476 tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
479 if f not in ssh_userkeys:
481 # If we're not exporting their primary group, don't export
484 if userlist[f] in grouprevmap.keys():
485 grname = grouprevmap[userlist[f]]
488 if int(userlist[f]) <= 100:
489 # In these cases, look it up in the normal way so we
490 # deal with cases where, for instance, users are in group
491 # users as their primary group.
492 grname = grp.getgrgid(userlist[f])[0]
497 print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f])
501 for line in ssh_userkeys[f]:
502 if line.startswith("allowed_hosts=") and ' ' in line:
503 machines, line = line.split('=', 1)[1].split(' ', 1)
504 if current_host not in machines.split(','):
505 continue # skip this key
508 continue # no keys for this host
509 contents = "\n".join(lines) + "\n"
511 to = tarfile.TarInfo(name=f)
512 # These will only be used where the username doesn't
513 # exist on the target system for some reason; hence,
514 # in those cases, the safest thing is for the file to
515 # be owned by root but group nobody. This deals with
516 # the bloody obscure case where the group fails to exist
517 # whilst the user does (in which case we want to avoid
518 # ending up with a file which is owned user:root to avoid
519 # a fairly obvious attack vector)
522 # Using the username / groupname fields avoids any need
523 # to give a shit^W^W^Wcare about the UIDoffset stuff.
527 to.mtime = int(time.time())
528 to.size = len(contents)
530 tf.addfile(to, StringIO(contents))
533 os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)
535 # add a list of groups to existing groups,
536 # including all subgroups thereof, recursively.
537 # basically this proceduces the transitive hull of the groups in
539 def addGroups(existingGroups, newGroups, uid, current_host):
540 for group in newGroups:
541 # if it's a <group>@host, split it and verify it's on the current host.
542 s = group.split('@', 1)
543 if len(s) == 2 and s[1] != current_host:
547 # let's see if we handled this group already
548 if group in existingGroups:
551 if not GroupIDMap.has_key(group):
552 print "Group", group, "does not exist but", uid, "is in it"
555 existingGroups.append(group)
557 if SubGroupMap.has_key(group):
558 addGroups(existingGroups, SubGroupMap[group], uid, current_host)
560 # Generate the group list
561 def GenGroup(accounts, File, current_host):
565 F = open(File + ".tdb.tmp", "w")
567 # Generate the GroupMap
571 GroupHasPrimaryMembers = {}
573 # Sort them into a list of groups having a set of users
575 GroupHasPrimaryMembers[ a['gidNumber'] ] = True
576 if not 'supplementaryGid' in a: continue
579 addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
581 GroupMap[g].append(a['uid'])
583 # Output the group file.
585 for x in GroupMap.keys():
586 if not x in GroupIDMap:
589 if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
592 grouprevmap[GroupIDMap[x]] = x
594 Line = "%s:x:%u:" % (x, GroupIDMap[x])
596 for I in GroupMap[x]:
597 Line = Line + ("%s%s" % (Comma, I))
599 Line = Sanitize(Line) + "\n"
600 F.write("0%u %s" % (J, Line))
601 F.write(".%s %s" % (x, Line))
602 F.write("=%u %s" % (GroupIDMap[x], Line))
605 # Oops, something unspeakable happened.
613 def CheckForward(accounts):
615 if not 'emailForward' in a: continue
619 # Do not allow people to try to buffer overflow busted parsers
620 if len(a['emailForward']) > 200: delete = True
621 # Check the forwarding address
622 elif EmailCheck.match(a['emailForward']) is None: delete = True
625 a.delete_mailforward()
627 # Generate the email forwarding list
628 def GenForward(accounts, File):
631 OldMask = os.umask(0022)
632 F = open(File + ".tmp", "w", 0644)
636 if not 'emailForward' in a: continue
637 Line = "%s: %s" % (a['uid'], a['emailForward'])
638 Line = Sanitize(Line) + "\n"
641 # Oops, something unspeakable happened.
647 def GenCDB(accounts, File, key):
650 OldMask = os.umask(0022)
651 # nothing else does the fsync stuff, so why do it here?
652 prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else ''
653 Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w")
656 # Write out the email address for each user
658 if not key in a: continue
661 Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
664 # Oops, something unspeakable happened.
668 if Fdb.close() != None:
669 raise "cdbmake gave an error"
671 def GenDBM(accounts, File, key):
673 OldMask = os.umask(0022)
674 fn = os.path.join(File).encode('ascii', 'ignore')
681 Fdb = dbm.open(fn, "c")
684 # Write out the email address for each user
686 if not key in a: continue
693 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
694 os.remove(File + ".db")
696 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
697 os.rename (File + ".db", File)
699 # Generate the anon XEarth marker file
700 def GenMarkers(accounts, File):
703 F = open(File + ".tmp", "w")
705 # Write out the position for each user
707 if not ('latitude' in a and 'longitude' in a): continue
709 Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True))
710 Line = Sanitize(Line) + "\n"
715 # Oops, something unspeakable happened.
721 # Generate the debian-private subscription list
722 def GenPrivate(accounts, File):
725 F = open(File + ".tmp", "w")
727 # Write out the position for each user
729 if not a.is_active_user(): continue
730 if a.is_guest_account(): continue
731 if not 'privateSub' in a: continue
733 Line = "%s"%(a['privateSub'])
734 Line = Sanitize(Line) + "\n"
739 # Oops, something unspeakable happened.
745 # Generate a list of locked accounts
746 def GenDisabledAccounts(accounts, File):
749 F = open(File + ".tmp", "w")
750 disabled_accounts = []
752 # Fetch all the users
754 if a.pw_active(): continue
755 Line = "%s:%s" % (a['uid'], "Account is locked")
756 disabled_accounts.append(a)
757 F.write(Sanitize(Line) + "\n")
759 # Oops, something unspeakable happened.
764 return disabled_accounts
766 # Generate the list of local addresses that refuse all mail
767 def GenMailDisable(accounts, File):
770 F = open(File + ".tmp", "w")
773 if not 'mailDisableMessage' in a: continue
774 Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
775 Line = Sanitize(Line) + "\n"
778 # Oops, something unspeakable happened.
784 # Generate a list of uids that should have boolean affects applied
785 def GenMailBool(accounts, File, key):
788 F = open(File + ".tmp", "w")
791 if not key in a: continue
792 if not a[key] == 'TRUE': continue
793 Line = "%s"%(a['uid'])
794 Line = Sanitize(Line) + "\n"
797 # Oops, something unspeakable happened.
803 # Generate a list of hosts for RBL or whitelist purposes.
804 def GenMailList(accounts, File, key):
807 F = open(File + ".tmp", "w")
809 if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$')
810 else: validregex = re.compile('^[-\w.]+$')
813 if not key in a: continue
815 filtered = filter(lambda z: validregex.match(z), a[key])
816 if len(filtered) == 0: continue
817 if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered)
818 line = a['uid'] + ': ' + ' : '.join(filtered)
819 line = Sanitize(line) + "\n"
822 # Oops, something unspeakable happened.
828 def isRoleAccount(account):
829 return 'debianRoleAccount' in account['objectClass']
831 # Generate the DNS Zone file
832 def GenDNS(accounts, File):
835 F = open(File + ".tmp", "w")
837 # Fetch all the users
840 # Write out the zone file entry for each user
842 if not 'dnsZoneEntry' in a: continue
843 if not a.is_active_user() and not isRoleAccount(a): continue
844 if a.is_guest_account(): continue
847 F.write("; %s\n"%(a.email_address()))
848 for z in a["dnsZoneEntry"]:
849 Split = z.lower().split()
850 if Split[1].lower() == 'in':
851 Line = " ".join(Split) + "\n"
854 Host = Split[0] + DNSZone
855 if BSMTPCheck.match(Line) != None:
856 F.write("; Has BSMTP\n")
858 # Write some identification information
859 if not RRs.has_key(Host):
860 if Split[2].lower() in ["a", "aaaa"]:
861 Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address())
862 for y in a["keyFingerPrint"]:
863 Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y))
867 Line = "; Err %s"%(str(Split))
872 F.write("; Errors:\n")
873 for line in str(e).split("\n"):
874 F.write("; %s\n"%(line))
877 # Oops, something unspeakable happened.
885 socket.inet_pton(socket.AF_INET6, i)
890 def ExtractDNSInfo(x):
891 hostname = GetAttr(x, "hostname")
895 TTLprefix="%s\t"%(x[1]["dnsTTL"][0])
898 if x[1].has_key("ipHostNumber"):
899 for I in x[1]["ipHostNumber"]:
901 DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I))
903 DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I))
907 ssh_hostnames = [ hostname ]
908 if x[1].has_key("sshfpHostname"):
909 ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ]
911 if 'sshRSAHostKey' in x[1]:
912 for I in x[1]["sshRSAHostKey"]:
914 key_prefix = Split[0]
915 key = base64.decodestring(Split[1])
918 # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
919 if key_prefix == 'ssh-rsa':
921 if key_prefix == 'ssh-dss':
923 if key_prefix == 'ssh-ed25519':
925 if Algorithm == None:
927 # and more from the registry
928 sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ]
930 fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ]
931 for h in ssh_hostnames:
932 for digest_codepoint, fingerprint in fingerprints:
933 DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))
935 if 'architecture' in x[1]:
936 Arch = GetAttr(x, "architecture")
938 if x[1].has_key("machine"):
939 Mach = " " + GetAttr(x, "machine")
940 DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))
942 if x[1].has_key("mXRecord"):
943 for I in x[1]["mXRecord"]:
945 for e in MX_remap[I]:
946 DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))
948 DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I))
952 # Generate the DNS records
953 def GenZoneRecords(host_attrs, File):
956 F = open(File + ".tmp", "w")
958 # Fetch all the hosts
960 if x[1].has_key("hostname") == 0:
963 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
966 for Line in ExtractDNSInfo(x):
969 # this would write sshfp lines for services on machines
970 # but we can't yet, since some are cnames and we'll make
971 # an invalid zonefile
973 # for i in x[1].get("purpose", []):
974 # m = PurposeHostField.match(i)
977 # # we ignore [[*..]] entries
978 # if m.startswith('*'):
980 # if m.startswith('-'):
983 # if not m.endswith(HostDomain):
985 # if not m.endswith('.'):
987 # for Line in DNSInfo:
988 # if isSSHFP.match(Line):
989 # Line = "%s\t%s" % (m, Line)
990 # F.write(Line + "\n")
992 # Oops, something unspeakable happened.
998 # Generate the BSMTP file
999 def GenBSMTP(accounts, File, HomePrefix):
1002 F = open(File + ".tmp", "w")
1004 # Write out the zone file entry for each user
1006 if not 'dnsZoneEntry' in a: continue
1007 if not a.is_active_user(): continue
1010 for z in a["dnsZoneEntry"]:
1011 Split = z.lower().split()
1012 if Split[1].lower() == 'in':
1013 for y in range(0, len(Split)):
1016 Line = " ".join(Split) + "\n"
1018 Host = Split[0] + DNSZone
1019 if BSMTPCheck.match(Line) != None:
1020 F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
1021 a['uid'], HomePrefix, a['uid'], Host))
1024 F.write("; Errors\n")
1027 # Oops, something unspeakable happened.
1033 def HostToIP(Host, mapped=True):
1037 if Host[1].has_key("ipHostNumber"):
1038 for addr in Host[1]["ipHostNumber"]:
1039 IPAdresses.append(addr)
1040 if not is_ipv6_addr(addr) and mapped == "True":
1041 IPAdresses.append("::ffff:"+addr)
1045 # Generate the ssh known hosts file
1046 def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
1049 OldMask = os.umask(0022)
1050 F = open(File + ".tmp", "w", 0644)
1053 for x in host_attrs:
1054 if x[1].has_key("hostname") == 0 or \
1055 x[1].has_key("sshRSAHostKey") == 0:
1057 Host = GetAttr(x, "hostname")
1058 HostNames = [ Host ]
1059 if Host.endswith(HostDomain):
1060 HostNames.append(Host[:-(len(HostDomain) + 1)])
1062 # in the purpose field [[host|some other text]] (where some other text is optional)
1063 # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
1064 # file. But so that we don't have to add everything we link we can add an asterisk
1065 # and say [[*... to ignore it. In order to be able to add stuff to ssh without
1066 # http linking it we also support [[-hostname]] entries.
1067 for i in x[1].get("purpose", []):
1068 m = PurposeHostField.match(i)
1071 # we ignore [[*..]] entries
1072 if m.startswith('*'):
1074 if m.startswith('-'):
1078 if m.endswith(HostDomain):
1079 HostNames.append(m[:-(len(HostDomain) + 1)])
1081 for I in x[1]["sshRSAHostKey"]:
1082 if mode and mode == 'authorized_keys':
1084 if 'sshdistAuthKeysHost' in x[1]:
1085 hosts += x[1]['sshdistAuthKeysHost']
1086 clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host)
1087 clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand)
1088 Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I)
1090 Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I)
1091 Line = Sanitize(Line) + "\n"
1093 # Oops, something unspeakable happened.
1099 # Generate the debianhosts file (list of all IP addresses)
1100 def GenHosts(host_attrs, File):
1103 OldMask = os.umask(0022)
1104 F = open(File + ".tmp", "w", 0644)
1109 for x in host_attrs:
1111 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
1114 if not 'ipHostNumber' in x[1]:
1117 addrs = x[1]["ipHostNumber"]
1119 if addr not in seen:
1121 addr = Sanitize(addr) + "\n"
1124 # Oops, something unspeakable happened.
1130 def replaceTree(src, dst_basedir):
1131 bn = os.path.basename(src)
1132 dst = os.path.join(dst_basedir, bn)
1134 shutil.copytree(src, dst)
1136 def GenKeyrings(OutDir):
1138 if os.path.isdir(k):
1139 replaceTree(k, OutDir)
1141 shutil.copy(k, OutDir)
1144 def get_accounts(ldap_conn):
1145 # Fetch all the users
1146 passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\
1147 ["uid", "uidNumber", "gidNumber", "supplementaryGid",\
1148 "gecos", "loginShell", "userPassword", "shadowLastChange",\
1149 "shadowMin", "shadowMax", "shadowWarning", "shadowInactive",
1150 "shadowExpire", "emailForward", "latitude", "longitude",\
1151 "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\
1152 "keyFingerPrint", "privateSub", "mailDisableMessage",\
1153 "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
1154 "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
1155 "mailContentInspectionAction", "webPassword", "rtcPassword",\
1156 "bATVToken", "totpSeed"])
1158 if passwd_attrs is None:
1159 raise UDEmptyList, "No Users"
1160 accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs)
1161 accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower()))
1165 def get_hosts(ldap_conn):
1166 # Fetch all the hosts
1167 HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
1168 ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
1169 "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
1172 if HostAttrs == None:
1173 raise UDEmptyList, "No Hosts"
1175 HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))
1180 def make_ldap_conn():
1181 # Connect to the ldap server
1183 # for testing purposes it's sometimes useful to pass username/password
1184 # via the environment
1185 if 'UD_CREDENTIALS' in os.environ:
1186 Pass = os.environ['UD_CREDENTIALS'].split()
1188 F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r")
1189 Pass = F.readline().strip().split(" ")
1191 l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1])
1197 def setup_group_maps(l):
1198 # Fetch all the groups
1201 attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
1202 ["gid", "gidNumber", "subGroup"])
1204 # Generate the subgroup_map and group_id_map
1206 if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":
1208 if x[1].has_key("gidNumber") == 0:
1210 group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
1211 if x[1].has_key("subGroup") != 0:
1212 subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])
1216 SubGroupMap = subgroup_map
1217 GroupIDMap = group_id_map
1219 def generate_all(global_dir, ldap_conn):
1220 accounts = get_accounts(ldap_conn)
1221 host_attrs = get_hosts(ldap_conn)
1224 # Generate global things
1225 accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
1227 accounts = filter(lambda x: not IsRetired(x), accounts)
1229 CheckForward(accounts)
1231 GenMailDisable(accounts, global_dir + "mail-disable")
1232 GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
1233 GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
1234 GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
1235 GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
1236 GenPrivate(accounts, global_dir + "debian-private")
1237 GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
1238 GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
1239 GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
1240 GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
1241 GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
1242 GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
1243 GenWebPassword(accounts, global_dir + "web-passwords")
1244 GenRtcPassword(accounts, global_dir + "rtc-passwords")
1245 GenTOTPSeed(accounts, global_dir + "users.oath")
1246 GenKeyrings(global_dir)
1249 GenForward(accounts, global_dir + "forward-alias")
1251 GenAllUsers(accounts, global_dir + 'all-accounts.json')
1252 accounts = filter(lambda a: not a in accounts_disabled, accounts)
1254 ssh_userkeys = GenSSHShadow(global_dir, accounts)
1255 GenMarkers(accounts, global_dir + "markers")
1256 GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
1257 GenHosts(host_attrs, global_dir + "debianhosts")
1259 GenDNS(accounts, global_dir + "dns-zone")
1260 GenZoneRecords(host_attrs, global_dir + "dns-sshfp")
1262 setup_group_maps(ldap_conn)
1264 for host in host_attrs:
1265 if not "hostname" in host[1]:
1267 generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
1269 def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
1270 current_host = host[1]['hostname'][0]
1271 OutDir = global_dir + current_host + '/'
1272 if not os.path.isdir(OutDir):
1275 # Get the group list and convert any named groups to numerics
1277 for groupname in AllowedGroupsPreload.strip().split(" "):
1278 GroupList[groupname] = True
1279 if 'allowedGroups' in host[1]:
1280 for groupname in host[1]['allowedGroups']:
1281 GroupList[groupname] = True
1282 for groupname in GroupList.keys():
1283 if groupname in GroupIDMap:
1284 GroupList[str(GroupIDMap[groupname])] = True
1287 if 'exportOptions' in host[1]:
1288 for extra in host[1]['exportOptions']:
1289 ExtraList[extra.upper()] = True
1292 accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)
1294 DoLink(global_dir, OutDir, "debianhosts")
1295 DoLink(global_dir, OutDir, "ssh_known_hosts")
1296 DoLink(global_dir, OutDir, "disabled-accounts")
1299 if 'NOPASSWD' in ExtraList:
1300 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*")
1302 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
1304 grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
1305 GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)
1307 # Now we know who we're allowing on the machine, export
1308 # the relevant ssh keys
1309 GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)
1311 if not 'NOPASSWD' in ExtraList:
1312 GenShadow(accounts, OutDir + "shadow")
1314 # Link in global things
1315 if not 'NOMARKERS' in ExtraList:
1316 DoLink(global_dir, OutDir, "markers")
1317 DoLink(global_dir, OutDir, "mail-forward.cdb")
1318 DoLink(global_dir, OutDir, "mail-forward.db")
1319 DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
1320 DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
1321 DoLink(global_dir, OutDir, "mail-disable")
1322 DoLink(global_dir, OutDir, "mail-greylist")
1323 DoLink(global_dir, OutDir, "mail-callout")
1324 DoLink(global_dir, OutDir, "mail-rbl")
1325 DoLink(global_dir, OutDir, "mail-rhsbl")
1326 DoLink(global_dir, OutDir, "mail-whitelist")
1327 DoLink(global_dir, OutDir, "all-accounts.json")
1328 GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
1329 GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
1330 GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
1331 GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')
1332 GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions')
1333 GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions')
1336 DoLink(global_dir, OutDir, "forward-alias")
1338 if 'DNS' in ExtraList:
1339 DoLink(global_dir, OutDir, "dns-zone")
1340 DoLink(global_dir, OutDir, "dns-sshfp")
1342 if 'AUTHKEYS' in ExtraList:
1343 DoLink(global_dir, OutDir, "authorized_keys")
1345 if 'BSMTP' in ExtraList:
1346 GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix)
1348 if 'PRIVATE' in ExtraList:
1349 DoLink(global_dir, OutDir, "debian-private")
1351 if 'GITOLITE' in ExtraList:
1352 GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)
1353 if 'exportOptions' in host[1]:
1354 for entry in host[1]['exportOptions']:
1355 v = entry.split('=',1)
1356 if v[0] != 'GITOLITE' or len(v) != 2: continue
1357 options = v[1].split(',')
1358 group = options.pop(0);
1359 gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
1360 if not 'nohosts' in options:
1361 gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
1366 if opt.startswith('sshcmd='):
1367 command = opt.split('=',1)[1]
1368 GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)
1370 if 'WEB-PASSWORDS' in ExtraList:
1371 DoLink(global_dir, OutDir, "web-passwords")
1373 if 'RTC-PASSWORDS' in ExtraList:
1374 DoLink(global_dir, OutDir, "rtc-passwords")
1376 if 'TOTP' in ExtraList:
1377 DoLink(global_dir, OutDir, "users.oath")
1379 if 'KEYRING' in ExtraList:
1381 bn = os.path.basename(k)
1382 if os.path.isdir(k):
1383 src = os.path.join(global_dir, bn)
1384 replaceTree(src, OutDir)
1386 DoLink(global_dir, OutDir, bn)
1390 bn = os.path.basename(k)
1391 target = os.path.join(OutDir, bn)
1392 if os.path.isdir(target):
1395 posix.remove(target)
1398 DoLink(global_dir, OutDir, "last_update.trace")
1401 def getLastLDAPChangeTime(l):
1402 mods = l.search_s('cn=log',
1403 ldap.SCOPE_ONELEVEL,
1404 '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))',
1409 # Sort the list by reqEnd
1410 sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0])
1411 # Take the last element in the array
1412 last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0]
1416 def getLastKeyringChangeTime():
1419 mt = os.path.getmtime(k)
1425 def getLastBuildTime(gdir):
1426 cache_last_ldap_mod = 0
1427 cache_last_unix_mod = 0
1431 fd = open(os.path.join(gdir, "last_update.trace"), "r")
1432 cache_last_mod=fd.read().split()
1434 cache_last_ldap_mod = cache_last_mod[0]
1435 cache_last_unix_mod = int(cache_last_mod[1])
1436 cache_last_run = int(cache_last_mod[2])
1437 except IndexError, ValueError:
1441 if e.errno == errno.ENOENT:
1446 return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
1448 def mq_notify(options, message):
1449 options.section = 'dsa-udgenerate'
1450 options.config = '/etc/dsa/pubsub.conf'
1452 config = Config(options)
1454 'rabbit_userid': config.username,
1455 'rabbit_password': config.password,
1456 'rabbit_virtual_host': config.vhost,
1457 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
1463 'timestamp': int(time.time())
1467 conn = Connection(conf=conf)
1468 conn.topic_send(config.topic,
1470 exchange_name=config.exchange,
1477 parser = optparse.OptionParser()
1478 parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
1479 help="Output directory.")
1480 parser.add_option("-f", "--force", dest="force", action="store_true",
1481 help="Force generation, even if no update to LDAP has happened.")
1483 (options, args) = parser.parse_args()
1488 if options.generatedir is not None:
1489 generate_dir = os.environ['UD_GENERATEDIR']
1490 elif 'UD_GENERATEDIR' in os.environ:
1491 generate_dir = os.environ['UD_GENERATEDIR']
1493 generate_dir = GenerateDir
1496 lockf = os.path.join(generate_dir, 'ud-generate.lock')
1497 lock = get_lock( lockf )
1499 sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
1502 l = make_ldap_conn()
1504 time_started = int(time.time())
1505 ldap_last_mod = getLastLDAPChangeTime(l)
1506 unix_last_mod = getLastKeyringChangeTime()
1507 cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)
1509 need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
1511 fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
1512 if need_update or options.force:
1513 msg = 'Update forced' if options.force else 'Update needed'
1514 generate_all(generate_dir, l)
1516 mq_notify(options, msg)
1517 last_run = int(time.time())
1518 fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
1523 if __name__ == "__main__":
1524 if 'UD_PROFILE' in os.environ:
1527 cProfile.run('ud_generate()', "udg_prof")
1528 p = pstats.Stats('udg_prof')
1529 ##p.sort_stats('time').print_stats()
1530 p.sort_stats('cumulative').print_stats()
1536 # vim:set shiftwidth=3: