3 # Generates passwd, shadow and group files from the ldap directory.
5 # Copyright (c) 2000-2001 Jason Gunthorpe <jgg@debian.org>
6 # Copyright (c) 2003-2004 James Troup <troup@debian.org>
7 # Copyright (c) 2004-2005,7 Joey Schulze <joey@infodrom.org>
8 # Copyright (c) 2001-2007 Ryan Murray <rmurray@debian.org>
9 # Copyright (c) 2008,2009,2010,2011 Peter Palfrader <peter@palfrader.org>
10 # Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
11 # Copyright (c) 2008 Mark Hymers <mhy@debian.org>
12 # Copyright (c) 2008 Luk Claes <luk@debian.org>
13 # Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>
14 # Copyright (c) 2009 Stephen Gran <steve@lobefin.net>
15 # Copyright (c) 2010 Helmut Grohne <helmut@subdivi.de>
17 # This program is free software; you can redistribute it and/or modify
18 # it under the terms of the GNU General Public License as published by
19 # the Free Software Foundation; either version 2 of the License, or
20 # (at your option) any later version.
22 # This program is distributed in the hope that it will be useful,
23 # but WITHOUT ANY WARRANTY; without even the implied warranty of
24 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 # GNU General Public License for more details.
27 # You should have received a copy of the GNU General Public License
28 # along with this program; if not, write to the Free Software
29 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
31 from dsa_mq.connection import Connection
32 from dsa_mq.config import Config
34 import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
36 from userdir_ldap import *
37 from userdir_exceptions import *
39 from xml.etree.ElementTree import Element, SubElement, Comment
40 from xml.etree import ElementTree
41 from xml.dom import minidom
43 from cStringIO import StringIO
45 from StringIO import StringIO
47 import simplejson as json
50 if '__author__' not in json.__dict__:
51 sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n")
52 sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n")
55 sys.stderr.write("You should probably not run ud-generate as root.\n")
67 UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
70 EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
71 BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL)
72 PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
73 IsDebianHost = re.compile(ConfModule.dns_hostmatch)
74 isSSHFP = re.compile("^\s*IN\s+SSHFP")
75 DNSZone = ".debian.net"
76 Keyrings = ConfModule.sync_keyrings.split(":")
77 GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
78 GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
79 GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
80 MX_remap = json.loads(ConfModule.MX_remap)
81 use_mq = getattr(ConfModule, "use_mq", True)
83 rtc_realm = getattr(ConfModule, "rtc_realm", None)
84 rtc_append = getattr(ConfModule, "rtc_append", None)
87 """Return a pretty-printed XML string for the Element.
89 rough_string = ElementTree.tostring(elem, 'utf-8')
90 reparsed = minidom.parseString(rough_string)
91 return reparsed.toprettyxml(indent=" ")
93 def safe_makedirs(dir):
97 if e.errno == errno.EEXIST:
102 def safe_rmtree(dir):
106 if e.errno == errno.ENOENT:
111 def get_lock(fn, wait=5*60):
114 ends = time.time() + wait
119 fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
123 if time.time() >= ends:
125 sl = min(sl*2, 10, ends - time.time())
131 return Str.translate(string.maketrans("\n\r\t", "$$$"))
133 def DoLink(From, To, File):
135 posix.remove(To + File)
138 posix.link(From + File, To + File)
140 def IsRetired(account):
142 Looks for accountStatus in the LDAP record and tries to
143 match it against one of the known retired statuses
146 status = account['accountStatus']
148 line = status.split()
151 if status == "inactive":
154 elif status == "memorial":
157 elif status == "retiring":
158 # We'll give them a few extra days over what we said
159 age = 6 * 31 * 24 * 60 * 60
161 return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age
169 # See if this user is in the group list
170 def IsInGroup(account, allowed, current_host):
171 # See if the primary group is in the list
172 if str(account['gidNumber']) in allowed: return True
174 # Check the host based ACL
175 if account.is_allowed_by_hostacl(current_host): return True
177 # See if there are supplementary groups
178 if 'supplementaryGid' not in account: return False
181 addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
187 def Die(File, F, Fdb):
193 os.remove(File + ".tmp")
197 os.remove(File + ".tdb.tmp")
201 def Done(File, F, Fdb):
204 os.rename(File + ".tmp", File)
207 os.rename(File + ".tdb.tmp", File + ".tdb")
209 # Generate the password list
210 def GenPasswd(accounts, File, HomePrefix, PwdMarker):
213 F = open(File + ".tdb.tmp", "w")
218 # Do not let people try to buffer overflow some busted passwd parser.
219 if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue
221 userlist[a['uid']] = a['gidNumber']
222 line = "%s:%s:%d:%d:%s:%s%s:%s" % (
228 HomePrefix, a['uid'],
230 line = Sanitize(line) + "\n"
231 F.write("0%u %s" % (i, line))
232 F.write(".%s %s" % (a['uid'], line))
233 F.write("=%d %s" % (a['uidNumber'], line))
236 # Oops, something unspeakable happened.
242 # Return the list of users so we know which keys to export
245 def GenAllUsers(accounts, file):
248 OldMask = os.umask(0022)
249 f = open(file + ".tmp", "w", 0644)
254 all.append( { 'uid': a['uid'],
255 'uidNumber': a['uidNumber'],
256 'active': a.pw_active() and a.shadow_active() } )
259 # Oops, something unspeakable happened.
265 # Generate the shadow list
266 def GenShadow(accounts, File):
269 OldMask = os.umask(0077)
270 F = open(File + ".tdb.tmp", "w", 0600)
275 # If the account is locked, mark it as such in shadow
276 # See Debian Bug #308229 for why we set it to 1 instead of 0
277 if not a.pw_active(): ShadowExpire = '1'
278 elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire'])
279 else: ShadowExpire = ''
282 values.append(a['uid'])
283 values.append(a.get_password())
284 for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive':
285 if key in a: values.append(a[key])
286 else: values.append('')
287 values.append(ShadowExpire)
288 line = ':'.join(values)+':'
289 line = Sanitize(line) + "\n"
290 F.write("0%u %s" % (i, line))
291 F.write(".%s %s" % (a['uid'], line))
294 # Oops, something unspeakable happened.
300 # Generate the sudo passwd file
301 def GenShadowSudo(accounts, File, untrusted, current_host):
304 OldMask = os.umask(0077)
305 F = open(File + ".tmp", "w", 0600)
310 if 'sudoPassword' in a:
311 for entry in a['sudoPassword']:
312 Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
315 uuid = Match.group(1)
316 status = Match.group(2)
317 hosts = Match.group(3)
318 cryptedpass = Match.group(4)
320 if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
322 for_all = hosts == "*"
323 for_this_host = current_host in hosts.split(',')
324 if not (for_all or for_this_host):
326 # ignore * passwords for untrusted hosts, but copy host specific passwords
327 if for_all and untrusted:
330 if for_this_host: # this makes sure we take a per-host entry over the for-all entry
335 Line = "%s:%s" % (a['uid'], Pass)
336 Line = Sanitize(Line) + "\n"
337 F.write("%s" % (Line))
339 # Oops, something unspeakable happened.
345 # Generate the sudo passwd file
346 def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
348 if sshcommand is None:
349 sshcommand = GitoliteSSHCommand
351 OldMask = os.umask(0022)
352 F = open(File + ".tmp", "w", 0600)
355 if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
357 if 'sshRSAAuthKey' not in a: continue
360 prefix = GitoliteSSHRestrictions
361 prefix = prefix.replace('@@COMMAND@@', sshcommand)
362 prefix = prefix.replace('@@USER@@', User)
363 for I in a["sshRSAAuthKey"]:
364 if I.startswith("allowed_hosts=") and ' ' in line:
365 if current_host is None:
367 machines, I = I.split('=', 1)[1].split(' ', 1)
368 if current_host not in machines.split(','):
369 continue # skip this key
371 if I.startswith('ssh-'):
372 line = "%s %s"%(prefix, I)
374 continue # do not allow keys with other restrictions that might conflict
375 line = Sanitize(line) + "\n"
378 for dn, attrs in hosts:
379 if 'sshRSAHostKey' not in attrs: continue
380 hostname = "host-" + attrs['hostname'][0]
381 prefix = GitoliteSSHRestrictions
382 prefix = prefix.replace('@@COMMAND@@', sshcommand)
383 prefix = prefix.replace('@@USER@@', hostname)
384 for I in attrs["sshRSAHostKey"]:
385 line = "%s %s"%(prefix, I)
386 line = Sanitize(line) + "\n"
389 # Oops, something unspeakable happened.
395 # Generate the shadow list
396 def GenSSHShadow(global_dir, accounts):
397 # Fetch all the users
401 if 'sshRSAAuthKey' not in a: continue
404 for I in a['sshRSAAuthKey']:
405 MultipleLine = "%s" % I
406 MultipleLine = Sanitize(MultipleLine)
407 contents.append(MultipleLine)
408 userkeys[a['uid']] = contents
411 # Generate the webPassword list
412 def GenWebPassword(accounts, File):
415 OldMask = os.umask(0077)
416 F = open(File, "w", 0600)
420 if 'webPassword' not in a: continue
421 if not a.pw_active(): continue
423 Pass = str(a['webPassword'])
424 Line = "%s:%s" % (a['uid'], Pass)
425 Line = Sanitize(Line) + "\n"
426 F.write("%s" % (Line))
432 # Generate the rtcPassword list
433 def GenRtcPassword(accounts, File):
436 OldMask = os.umask(0077)
437 F = open(File, "w", 0600)
441 if a.is_guest_account(): continue
442 if 'rtcPassword' not in a: continue
443 if not a.pw_active(): continue
445 Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
446 Line = Sanitize(Line) + "\n"
447 F.write("%s" % (Line))
453 # Generate the TOTP auth file
454 def GenTOTPSeed(accounts, File):
457 OldMask = os.umask(0077)
458 F = open(File, "w", 0600)
461 F.write("# Option User Prefix Seed\n")
463 if a.is_guest_account(): continue
464 if 'totpSeed' not in a: continue
465 if not a.pw_active(): continue
467 Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
468 Line = Sanitize(Line) + "\n"
469 F.write("%s" % (Line))
475 def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
476 OldMask = os.umask(0077)
477 tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
480 if f not in ssh_userkeys:
482 # If we're not exporting their primary group, don't export
485 if userlist[f] in grouprevmap.keys():
486 grname = grouprevmap[userlist[f]]
489 if int(userlist[f]) <= 100:
490 # In these cases, look it up in the normal way so we
491 # deal with cases where, for instance, users are in group
492 # users as their primary group.
493 grname = grp.getgrgid(userlist[f])[0]
498 print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f])
502 for line in ssh_userkeys[f]:
503 if line.startswith("allowed_hosts=") and ' ' in line:
504 machines, line = line.split('=', 1)[1].split(' ', 1)
505 if current_host not in machines.split(','):
506 continue # skip this key
509 continue # no keys for this host
510 contents = "\n".join(lines) + "\n"
512 to = tarfile.TarInfo(name=f)
513 # These will only be used where the username doesn't
514 # exist on the target system for some reason; hence,
515 # in those cases, the safest thing is for the file to
516 # be owned by root but group nobody. This deals with
517 # the bloody obscure case where the group fails to exist
518 # whilst the user does (in which case we want to avoid
519 # ending up with a file which is owned user:root to avoid
520 # a fairly obvious attack vector)
523 # Using the username / groupname fields avoids any need
524 # to give a shit^W^W^Wcare about the UIDoffset stuff.
528 to.mtime = int(time.time())
529 to.size = len(contents)
531 tf.addfile(to, StringIO(contents))
534 os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)
536 # add a list of groups to existing groups,
537 # including all subgroups thereof, recursively.
538 # basically this proceduces the transitive hull of the groups in
540 def addGroups(existingGroups, newGroups, uid, current_host):
541 for group in newGroups:
542 # if it's a <group>@host, split it and verify it's on the current host.
543 s = group.split('@', 1)
544 if len(s) == 2 and s[1] != current_host:
548 # let's see if we handled this group already
549 if group in existingGroups:
552 if not GroupIDMap.has_key(group):
553 print "Group", group, "does not exist but", uid, "is in it"
556 existingGroups.append(group)
558 if SubGroupMap.has_key(group):
559 addGroups(existingGroups, SubGroupMap[group], uid, current_host)
561 # Generate the group list
562 def GenGroup(accounts, File, current_host):
566 F = open(File + ".tdb.tmp", "w")
568 # Generate the GroupMap
572 GroupHasPrimaryMembers = {}
574 # Sort them into a list of groups having a set of users
576 GroupHasPrimaryMembers[ a['gidNumber'] ] = True
577 if 'supplementaryGid' not in a: continue
580 addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
582 GroupMap[g].append(a['uid'])
584 # Output the group file.
586 for x in GroupMap.keys():
587 if x not in GroupIDMap:
590 if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
593 grouprevmap[GroupIDMap[x]] = x
595 Line = "%s:x:%u:" % (x, GroupIDMap[x])
597 for I in GroupMap[x]:
598 Line = Line + ("%s%s" % (Comma, I))
600 Line = Sanitize(Line) + "\n"
601 F.write("0%u %s" % (J, Line))
602 F.write(".%s %s" % (x, Line))
603 F.write("=%u %s" % (GroupIDMap[x], Line))
606 # Oops, something unspeakable happened.
614 def CheckForward(accounts):
616 if 'emailForward' not in a: continue
620 # Do not allow people to try to buffer overflow busted parsers
621 if len(a['emailForward']) > 200: delete = True
622 # Check the forwarding address
623 elif EmailCheck.match(a['emailForward']) is None: delete = True
626 a.delete_mailforward()
628 # Generate the email forwarding list
629 def GenForward(accounts, File):
632 OldMask = os.umask(0022)
633 F = open(File + ".tmp", "w", 0644)
637 if 'emailForward' not in a: continue
638 Line = "%s: %s" % (a['uid'], a['emailForward'])
639 Line = Sanitize(Line) + "\n"
642 # Oops, something unspeakable happened.
648 def GenCDB(accounts, File, key):
649 prefix = ["/usr/bin/eatmydata"] if os.path.exists('/usr/bin/eatmydata') else []
650 # nothing else does the fsync stuff, so why do it here?
651 Fdb = subprocess.Popen(prefix + ["cdbmake", File, "%s.tmp" % File],
652 preexec_fn=lambda: os.umask(0022),
653 stdin=subprocess.PIPE)
655 # Write out the email address for each user
657 if key not in a: continue
660 Fdb.stdin.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
662 Fdb.stdin.write("\n")
666 raise Exception("cdbmake gave an error")
668 def GenDBM(accounts, File, key):
670 OldMask = os.umask(0022)
671 fn = os.path.join(File).encode('ascii', 'ignore')
678 Fdb = dbm.open(fn, "c")
681 # Write out the email address for each user
683 if key not in a: continue
690 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
691 os.remove(File + ".db")
693 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
694 os.rename (File + ".db", File)
696 # Generate the anon XEarth marker file
697 def GenMarkers(accounts, File):
700 F = open(File + ".tmp", "w")
702 # Write out the position for each user
704 if not ('latitude' in a and 'longitude' in a): continue
706 Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True))
707 Line = Sanitize(Line) + "\n"
712 # Oops, something unspeakable happened.
718 # Generate the debian-private subscription list
719 def GenPrivate(accounts, File):
722 F = open(File + ".tmp", "w")
724 # Write out the position for each user
726 if not a.is_active_user(): continue
727 if a.is_guest_account(): continue
728 if 'privateSub' not in a: continue
730 Line = "%s"%(a['privateSub'])
731 Line = Sanitize(Line) + "\n"
736 # Oops, something unspeakable happened.
742 # Generate a list of locked accounts
743 def GenDisabledAccounts(accounts, File):
746 F = open(File + ".tmp", "w")
747 disabled_accounts = []
749 # Fetch all the users
751 if a.pw_active(): continue
752 Line = "%s:%s" % (a['uid'], "Account is locked")
753 disabled_accounts.append(a)
754 F.write(Sanitize(Line) + "\n")
756 # Oops, something unspeakable happened.
761 return disabled_accounts
763 # Generate the list of local addresses that refuse all mail
764 def GenMailDisable(accounts, File):
767 F = open(File + ".tmp", "w")
770 if 'mailDisableMessage' not in a: continue
771 Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
772 Line = Sanitize(Line) + "\n"
775 # Oops, something unspeakable happened.
781 # Generate a list of uids that should have boolean affects applied
782 def GenMailBool(accounts, File, key):
785 F = open(File + ".tmp", "w")
788 if key not in a: continue
789 if not a[key] == 'TRUE': continue
790 Line = "%s"%(a['uid'])
791 Line = Sanitize(Line) + "\n"
794 # Oops, something unspeakable happened.
800 # Generate a list of hosts for RBL or whitelist purposes.
801 def GenMailList(accounts, File, key):
804 F = open(File + ".tmp", "w")
806 if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$')
807 else: validregex = re.compile('^[-\w.]+$')
810 if key not in a: continue
812 filtered = filter(lambda z: validregex.match(z), a[key])
813 if len(filtered) == 0: continue
814 if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered)
815 line = a['uid'] + ': ' + ' : '.join(filtered)
816 line = Sanitize(line) + "\n"
819 # Oops, something unspeakable happened.
825 def isRoleAccount(account):
826 return 'debianRoleAccount' in account['objectClass']
828 # Generate the DNS Zone file
829 def GenDNS(accounts, File):
832 F = open(File + ".tmp", "w")
834 # Fetch all the users
837 # Write out the zone file entry for each user
839 if 'dnsZoneEntry' not in a: continue
840 if not a.is_active_user() and not isRoleAccount(a): continue
841 if a.is_guest_account(): continue
844 F.write("; %s\n"%(a.email_address()))
845 for z in a["dnsZoneEntry"]:
846 Split = z.lower().split()
847 if Split[1].lower() == 'in':
848 Line = " ".join(Split) + "\n"
851 Host = Split[0] + DNSZone
852 if BSMTPCheck.match(Line) is not None:
853 F.write("; Has BSMTP\n")
855 # Write some identification information
856 if not RRs.has_key(Host):
857 if Split[2].lower() in ["a", "aaaa"]:
858 Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address())
859 for y in a["keyFingerPrint"]:
860 Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y))
864 Line = "; Err %s"%(str(Split))
869 F.write("; Errors:\n")
870 for line in str(e).split("\n"):
871 F.write("; %s\n"%(line))
874 # Oops, something unspeakable happened.
882 socket.inet_pton(socket.AF_INET6, i)
887 def ExtractDNSInfo(x):
888 hostname = GetAttr(x, "hostname")
892 TTLprefix="%s\t"%(x[1]["dnsTTL"][0])
895 if x[1].has_key("ipHostNumber"):
896 for I in x[1]["ipHostNumber"]:
898 DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I))
900 DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I))
904 ssh_hostnames = [ hostname ]
905 if x[1].has_key("sshfpHostname"):
906 ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ]
908 if 'sshRSAHostKey' in x[1]:
909 for I in x[1]["sshRSAHostKey"]:
911 key_prefix = Split[0]
912 key = base64.decodestring(Split[1])
915 # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
916 if key_prefix == 'ssh-rsa':
918 if key_prefix == 'ssh-dss':
920 if key_prefix == 'ssh-ed25519':
922 if Algorithm is None:
924 # and more from the registry
925 sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ]
927 fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ]
928 for h in ssh_hostnames:
929 for digest_codepoint, fingerprint in fingerprints:
930 DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))
932 if 'architecture' in x[1]:
933 Arch = GetAttr(x, "architecture")
935 if x[1].has_key("machine"):
936 Mach = " " + GetAttr(x, "machine")
937 DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))
939 if x[1].has_key("mXRecord"):
940 for I in x[1]["mXRecord"]:
942 for e in MX_remap[I]:
943 DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))
945 DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I))
949 # Generate the DNS records
950 def GenZoneRecords(host_attrs, File):
953 F = open(File + ".tmp", "w")
955 # Fetch all the hosts
957 if x[1].has_key("hostname") == 0:
960 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
963 for Line in ExtractDNSInfo(x):
966 # Oops, something unspeakable happened.
972 # Generate the BSMTP file
973 def GenBSMTP(accounts, File, HomePrefix):
976 F = open(File + ".tmp", "w")
978 # Write out the zone file entry for each user
980 if 'dnsZoneEntry' not in a: continue
981 if not a.is_active_user(): continue
984 for z in a["dnsZoneEntry"]:
985 Split = z.lower().split()
986 if Split[1].lower() == 'in':
987 for y in range(0, len(Split)):
990 Line = " ".join(Split) + "\n"
992 Host = Split[0] + DNSZone
993 if BSMTPCheck.match(Line) is not None:
994 F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
995 a['uid'], HomePrefix, a['uid'], Host))
998 F.write("; Errors\n")
1001 # Oops, something unspeakable happened.
1007 def HostToIP(Host, mapped=True):
1011 if Host[1].has_key("ipHostNumber"):
1012 for addr in Host[1]["ipHostNumber"]:
1013 IPAdresses.append(addr)
1014 if not is_ipv6_addr(addr) and mapped == "True":
1015 IPAdresses.append("::ffff:"+addr)
1019 # Generate the ssh known hosts file
1020 def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
1023 OldMask = os.umask(0022)
1024 F = open(File + ".tmp", "w", 0644)
1027 for x in host_attrs:
1028 if x[1].has_key("hostname") == 0 or \
1029 x[1].has_key("sshRSAHostKey") == 0:
1031 Host = GetAttr(x, "hostname")
1032 HostNames = [ Host ]
1033 if Host.endswith(HostDomain):
1034 HostNames.append(Host[:-(len(HostDomain) + 1)])
1036 # in the purpose field [[host|some other text]] (where some other text is optional)
1037 # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
1038 # file. But so that we don't have to add everything we link we can add an asterisk
1039 # and say [[*... to ignore it. In order to be able to add stuff to ssh without
1040 # http linking it we also support [[-hostname]] entries.
1041 for i in x[1].get("purpose", []):
1042 m = PurposeHostField.match(i)
1045 # we ignore [[*..]] entries
1046 if m.startswith('*'):
1048 if m.startswith('-'):
1052 if m.endswith(HostDomain):
1053 HostNames.append(m[:-(len(HostDomain) + 1)])
1055 for I in x[1]["sshRSAHostKey"]:
1056 if mode and mode == 'authorized_keys':
1058 if 'sshdistAuthKeysHost' in x[1]:
1059 hosts += x[1]['sshdistAuthKeysHost']
1060 clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host)
1061 clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand)
1062 Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I)
1064 Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I)
1065 Line = Sanitize(Line) + "\n"
1067 # Oops, something unspeakable happened.
1073 # Generate the debianhosts file (list of all IP addresses)
1074 def GenHosts(host_attrs, File):
1077 OldMask = os.umask(0022)
1078 F = open(File + ".tmp", "w", 0644)
1083 for x in host_attrs:
1085 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
1088 if 'ipHostNumber' not in x[1]:
1091 addrs = x[1]["ipHostNumber"]
1093 if addr not in seen:
1095 addr = Sanitize(addr) + "\n"
1098 # Oops, something unspeakable happened.
1104 def replaceTree(src, dst_basedir):
1105 bn = os.path.basename(src)
1106 dst = os.path.join(dst_basedir, bn)
1108 shutil.copytree(src, dst)
1110 def GenKeyrings(OutDir):
1112 if os.path.isdir(k):
1113 replaceTree(k, OutDir)
1115 shutil.copy(k, OutDir)
1118 def get_accounts(ldap_conn):
1119 # Fetch all the users
1120 passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\
1121 ["uid", "uidNumber", "gidNumber", "supplementaryGid",\
1122 "gecos", "loginShell", "userPassword", "shadowLastChange",\
1123 "shadowMin", "shadowMax", "shadowWarning", "shadowInactive",
1124 "shadowExpire", "emailForward", "latitude", "longitude",\
1125 "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\
1126 "keyFingerPrint", "privateSub", "mailDisableMessage",\
1127 "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
1128 "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
1129 "mailContentInspectionAction", "webPassword", "rtcPassword",\
1130 "bATVToken", "totpSeed", "mailDefaultOptions"])
1132 if passwd_attrs is None:
1133 raise UDEmptyList, "No Users"
1134 accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs)
1135 accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower()))
1139 def get_hosts(ldap_conn):
1140 # Fetch all the hosts
1141 HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
1142 ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
1143 "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
1146 if HostAttrs is None:
1147 raise UDEmptyList, "No Hosts"
1149 HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))
1154 def make_ldap_conn():
1155 # Connect to the ldap server
1157 # for testing purposes it's sometimes useful to pass username/password
1158 # via the environment
1159 if 'UD_CREDENTIALS' in os.environ:
1160 Pass = os.environ['UD_CREDENTIALS'].split()
1162 F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r")
1163 Pass = F.readline().strip().split(" ")
1165 l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1])
1171 def setup_group_maps(l):
1172 # Fetch all the groups
1175 attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
1176 ["gid", "gidNumber", "subGroup"])
1178 # Generate the subgroup_map and group_id_map
1180 if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":
1182 if x[1].has_key("gidNumber") == 0:
1184 group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
1185 if x[1].has_key("subGroup") != 0:
1186 subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])
1190 SubGroupMap = subgroup_map
1191 GroupIDMap = group_id_map
1193 def generate_all(global_dir, ldap_conn):
1194 accounts = get_accounts(ldap_conn)
1195 host_attrs = get_hosts(ldap_conn)
1198 # Generate global things
1199 accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
1201 accounts = filter(lambda x: not IsRetired(x), accounts)
1203 CheckForward(accounts)
1205 GenMailDisable(accounts, global_dir + "mail-disable")
1206 GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
1207 GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
1208 GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
1209 GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
1210 GenCDB(accounts, global_dir + "default-mail-options.cdb", 'mailDefaultOptions')
1211 GenDBM(accounts, global_dir + "default-mail-options.db", 'mailDefaultOptions')
1212 GenPrivate(accounts, global_dir + "debian-private")
1213 GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
1214 GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
1215 GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
1216 GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
1217 GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
1218 GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
1219 GenWebPassword(accounts, global_dir + "web-passwords")
1220 GenRtcPassword(accounts, global_dir + "rtc-passwords")
1221 GenTOTPSeed(accounts, global_dir + "users.oath")
1222 GenKeyrings(global_dir)
1225 GenForward(accounts, global_dir + "forward-alias")
1227 GenAllUsers(accounts, global_dir + 'all-accounts.json')
1228 accounts = filter(lambda a: a not in accounts_disabled, accounts)
1230 ssh_userkeys = GenSSHShadow(global_dir, accounts)
1231 GenMarkers(accounts, global_dir + "markers")
1232 GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
1233 GenHosts(host_attrs, global_dir + "debianhosts")
1235 GenDNS(accounts, global_dir + "dns-zone")
1236 GenZoneRecords(host_attrs, global_dir + "dns-sshfp")
1238 setup_group_maps(ldap_conn)
1240 for host in host_attrs:
1241 if "hostname" not in host[1]:
1243 generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
1245 def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
1246 current_host = host[1]['hostname'][0]
1247 OutDir = global_dir + current_host + '/'
1248 if not os.path.isdir(OutDir):
1251 # Get the group list and convert any named groups to numerics
1253 for groupname in AllowedGroupsPreload.strip().split(" "):
1254 GroupList[groupname] = True
1255 if 'allowedGroups' in host[1]:
1256 for groupname in host[1]['allowedGroups']:
1257 GroupList[groupname] = True
1258 for groupname in GroupList.keys():
1259 if groupname in GroupIDMap:
1260 GroupList[str(GroupIDMap[groupname])] = True
1263 if 'exportOptions' in host[1]:
1264 for extra in host[1]['exportOptions']:
1265 ExtraList[extra.upper()] = True
1268 accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)
1270 DoLink(global_dir, OutDir, "debianhosts")
1271 DoLink(global_dir, OutDir, "ssh_known_hosts")
1272 DoLink(global_dir, OutDir, "disabled-accounts")
1275 if 'NOPASSWD' in ExtraList:
1276 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*")
1278 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
1280 grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
1281 GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)
1283 # Now we know who we're allowing on the machine, export
1284 # the relevant ssh keys
1285 GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)
1287 if 'NOPASSWD' not in ExtraList:
1288 GenShadow(accounts, OutDir + "shadow")
1290 # Link in global things
1291 if 'NOMARKERS' not in ExtraList:
1292 DoLink(global_dir, OutDir, "markers")
1293 DoLink(global_dir, OutDir, "mail-forward.cdb")
1294 DoLink(global_dir, OutDir, "mail-forward.db")
1295 DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
1296 DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
1297 DoLink(global_dir, OutDir, "mail-disable")
1298 DoLink(global_dir, OutDir, "mail-greylist")
1299 DoLink(global_dir, OutDir, "mail-callout")
1300 DoLink(global_dir, OutDir, "mail-rbl")
1301 DoLink(global_dir, OutDir, "mail-rhsbl")
1302 DoLink(global_dir, OutDir, "mail-whitelist")
1303 DoLink(global_dir, OutDir, "all-accounts.json")
1304 DoLink(global_dir, OutDir, "default-mail-options.cdb")
1305 DoLink(global_dir, OutDir, "default-mail-options.db")
1306 GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
1307 GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
1308 GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
1309 GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')
1312 DoLink(global_dir, OutDir, "forward-alias")
1314 if 'DNS' in ExtraList:
1315 DoLink(global_dir, OutDir, "dns-zone")
1316 DoLink(global_dir, OutDir, "dns-sshfp")
1318 if 'AUTHKEYS' in ExtraList:
1319 DoLink(global_dir, OutDir, "authorized_keys")
1321 if 'BSMTP' in ExtraList:
1322 GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix)
1324 if 'PRIVATE' in ExtraList:
1325 DoLink(global_dir, OutDir, "debian-private")
1327 if 'GITOLITE' in ExtraList:
1328 GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)
1329 if 'exportOptions' in host[1]:
1330 for entry in host[1]['exportOptions']:
1331 v = entry.split('=',1)
1332 if v[0] != 'GITOLITE' or len(v) != 2: continue
1333 options = v[1].split(',')
1334 group = options.pop(0)
1335 gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
1336 if 'nohosts' not in options:
1337 gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
1342 if opt.startswith('sshcmd='):
1343 command = opt.split('=',1)[1]
1344 GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)
1346 if 'WEB-PASSWORDS' in ExtraList:
1347 DoLink(global_dir, OutDir, "web-passwords")
1349 if 'RTC-PASSWORDS' in ExtraList:
1350 DoLink(global_dir, OutDir, "rtc-passwords")
1352 if 'TOTP' in ExtraList:
1353 DoLink(global_dir, OutDir, "users.oath")
1355 if 'KEYRING' in ExtraList:
1357 bn = os.path.basename(k)
1358 if os.path.isdir(k):
1359 src = os.path.join(global_dir, bn)
1360 replaceTree(src, OutDir)
1362 DoLink(global_dir, OutDir, bn)
1366 bn = os.path.basename(k)
1367 target = os.path.join(OutDir, bn)
1368 if os.path.isdir(target):
1371 posix.remove(target)
1374 DoLink(global_dir, OutDir, "last_update.trace")
1377 def getLastLDAPChangeTime(l):
1378 mods = l.search_s('cn=log',
1379 ldap.SCOPE_ONELEVEL,
1380 '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))',
1385 # Sort the list by reqEnd
1386 sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0])
1387 # Take the last element in the array
1388 last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0]
1392 def getLastKeyringChangeTime():
1395 mt = os.path.getmtime(k)
1401 def getLastBuildTime(gdir):
1402 cache_last_ldap_mod = 0
1403 cache_last_unix_mod = 0
1407 fd = open(os.path.join(gdir, "last_update.trace"), "r")
1408 cache_last_mod=fd.read().split()
1410 cache_last_ldap_mod = cache_last_mod[0]
1411 cache_last_unix_mod = int(cache_last_mod[1])
1412 cache_last_run = int(cache_last_mod[2])
1413 except IndexError, ValueError:
1417 if e.errno == errno.ENOENT:
1422 return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
1424 def mq_notify(options, message):
1425 options.section = 'dsa-udgenerate'
1426 options.config = '/etc/dsa/pubsub.conf'
1428 config = Config(options)
1430 'rabbit_userid': config.username,
1431 'rabbit_password': config.password,
1432 'rabbit_virtual_host': config.vhost,
1433 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
1439 'timestamp': int(time.time())
1443 conn = Connection(conf=conf)
1444 conn.topic_send(config.topic,
1446 exchange_name=config.exchange,
1453 parser = optparse.OptionParser()
1454 parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
1455 help="Output directory.")
1456 parser.add_option("-f", "--force", dest="force", action="store_true",
1457 help="Force generation, even if no update to LDAP has happened.")
1459 (options, args) = parser.parse_args()
1464 if options.generatedir is not None:
1465 generate_dir = os.environ['UD_GENERATEDIR']
1466 elif 'UD_GENERATEDIR' in os.environ:
1467 generate_dir = os.environ['UD_GENERATEDIR']
1469 generate_dir = GenerateDir
1472 lockf = os.path.join(generate_dir, 'ud-generate.lock')
1473 lock = get_lock( lockf )
1475 sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
1478 l = make_ldap_conn()
1480 time_started = int(time.time())
1481 ldap_last_mod = getLastLDAPChangeTime(l)
1482 unix_last_mod = getLastKeyringChangeTime()
1483 cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)
1485 need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
1487 fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
1488 if need_update or options.force:
1489 msg = 'Update forced' if options.force else 'Update needed'
1490 generate_all(generate_dir, l)
1492 mq_notify(options, msg)
1493 last_run = int(time.time())
1494 fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
1499 if __name__ == "__main__":
1500 if 'UD_PROFILE' in os.environ:
1503 cProfile.run('ud_generate()', "udg_prof")
1504 p = pstats.Stats('udg_prof')
1505 ##p.sort_stats('time').print_stats()
1506 p.sort_stats('cumulative').print_stats()
1512 # vim:set shiftwidth=3: