3 # Generates passwd, shadow and group files from the ldap directory.
5 # Copyright (c) 2000-2001 Jason Gunthorpe <jgg@debian.org>
6 # Copyright (c) 2003-2004 James Troup <troup@debian.org>
7 # Copyright (c) 2004-2005,7 Joey Schulze <joey@infodrom.org>
8 # Copyright (c) 2001-2007 Ryan Murray <rmurray@debian.org>
9 # Copyright (c) 2008,2009,2010,2011 Peter Palfrader <peter@palfrader.org>
10 # Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
11 # Copyright (c) 2008 Mark Hymers <mhy@debian.org>
12 # Copyright (c) 2008 Luk Claes <luk@debian.org>
13 # Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>
14 # Copyright (c) 2009 Stephen Gran <steve@lobefin.net>
15 # Copyright (c) 2010 Helmut Grohne <helmut@subdivi.de>
17 # This program is free software; you can redistribute it and/or modify
18 # it under the terms of the GNU General Public License as published by
19 # the Free Software Foundation; either version 2 of the License, or
20 # (at your option) any later version.
22 # This program is distributed in the hope that it will be useful,
23 # but WITHOUT ANY WARRANTY; without even the implied warranty of
24 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 # GNU General Public License for more details.
27 # You should have received a copy of the GNU General Public License
28 # along with this program; if not, write to the Free Software
29 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
31 from dsa_mq.connection import Connection
32 from dsa_mq.config import Config
34 import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
35 from userdir_ldap import *
36 from userdir_exceptions import *
38 from xml.etree.ElementTree import Element, SubElement, Comment
39 from xml.etree import ElementTree
40 from xml.dom import minidom
42 from cStringIO import StringIO
44 from StringIO import StringIO
46 import simplejson as json
49 if '__author__' not in json.__dict__:
50 sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n")
51 sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n")
54 sys.stderr.write("You should probably not run ud-generate as root.\n")
66 UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
69 EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
70 BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL)
71 PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
72 IsDebianHost = re.compile(ConfModule.dns_hostmatch)
73 isSSHFP = re.compile("^\s*IN\s+SSHFP")
74 DNSZone = ".debian.net"
75 Keyrings = ConfModule.sync_keyrings.split(":")
76 GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
77 GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
78 GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
79 MX_remap = json.loads(ConfModule.MX_remap)
80 use_mq = getattr(ConfModule, "use_mq", True)
82 rtc_realm = getattr(ConfModule, "rtc_realm", None)
83 rtc_append = getattr(ConfModule, "rtc_append", None)
86 """Return a pretty-printed XML string for the Element.
88 rough_string = ElementTree.tostring(elem, 'utf-8')
89 reparsed = minidom.parseString(rough_string)
90 return reparsed.toprettyxml(indent=" ")
92 def safe_makedirs(dir):
96 if e.errno == errno.EEXIST:
101 def safe_rmtree(dir):
105 if e.errno == errno.ENOENT:
110 def get_lock(fn, wait=5*60):
113 ends = time.time() + wait
118 fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
122 if time.time() >= ends:
124 sl = min(sl*2, 10, ends - time.time())
130 return Str.translate(string.maketrans("\n\r\t", "$$$"))
132 def DoLink(From, To, File):
134 posix.remove(To + File)
137 posix.link(From + File, To + File)
139 def IsRetired(account):
141 Looks for accountStatus in the LDAP record and tries to
142 match it against one of the known retired statuses
145 status = account['accountStatus']
147 line = status.split()
150 if status == "inactive":
153 elif status == "memorial":
156 elif status == "retiring":
157 # We'll give them a few extra days over what we said
158 age = 6 * 31 * 24 * 60 * 60
160 return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age
168 # See if this user is in the group list
169 def IsInGroup(account, allowed, current_host):
170 # See if the primary group is in the list
171 if str(account['gidNumber']) in allowed: return True
173 # Check the host based ACL
174 if account.is_allowed_by_hostacl(current_host): return True
176 # See if there are supplementary groups
177 if 'supplementaryGid' not in account: return False
180 addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
186 def Die(File, F, Fdb):
192 os.remove(File + ".tmp")
196 os.remove(File + ".tdb.tmp")
200 def Done(File, F, Fdb):
203 os.rename(File + ".tmp", File)
206 os.rename(File + ".tdb.tmp", File + ".tdb")
208 # Generate the password list
209 def GenPasswd(accounts, File, HomePrefix, PwdMarker):
212 F = open(File + ".tdb.tmp", "w")
217 # Do not let people try to buffer overflow some busted passwd parser.
218 if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue
220 userlist[a['uid']] = a['gidNumber']
221 line = "%s:%s:%d:%d:%s:%s%s:%s" % (
227 HomePrefix, a['uid'],
229 line = Sanitize(line) + "\n"
230 F.write("0%u %s" % (i, line))
231 F.write(".%s %s" % (a['uid'], line))
232 F.write("=%d %s" % (a['uidNumber'], line))
235 # Oops, something unspeakable happened.
241 # Return the list of users so we know which keys to export
244 def GenAllUsers(accounts, file):
247 OldMask = os.umask(0022)
248 f = open(file + ".tmp", "w", 0644)
253 all.append( { 'uid': a['uid'],
254 'uidNumber': a['uidNumber'],
255 'active': a.pw_active() and a.shadow_active() } )
258 # Oops, something unspeakable happened.
264 # Generate the shadow list
265 def GenShadow(accounts, File):
268 OldMask = os.umask(0077)
269 F = open(File + ".tdb.tmp", "w", 0600)
274 # If the account is locked, mark it as such in shadow
275 # See Debian Bug #308229 for why we set it to 1 instead of 0
276 if not a.pw_active(): ShadowExpire = '1'
277 elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire'])
278 else: ShadowExpire = ''
281 values.append(a['uid'])
282 values.append(a.get_password())
283 for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive':
284 if key in a: values.append(a[key])
285 else: values.append('')
286 values.append(ShadowExpire)
287 line = ':'.join(values)+':'
288 line = Sanitize(line) + "\n"
289 F.write("0%u %s" % (i, line))
290 F.write(".%s %s" % (a['uid'], line))
293 # Oops, something unspeakable happened.
299 # Generate the sudo passwd file
300 def GenShadowSudo(accounts, File, untrusted, current_host):
303 OldMask = os.umask(0077)
304 F = open(File + ".tmp", "w", 0600)
309 if 'sudoPassword' in a:
310 for entry in a['sudoPassword']:
311 Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
314 uuid = Match.group(1)
315 status = Match.group(2)
316 hosts = Match.group(3)
317 cryptedpass = Match.group(4)
319 if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
321 for_all = hosts == "*"
322 for_this_host = current_host in hosts.split(',')
323 if not (for_all or for_this_host):
325 # ignore * passwords for untrusted hosts, but copy host specific passwords
326 if for_all and untrusted:
329 if for_this_host: # this makes sure we take a per-host entry over the for-all entry
334 Line = "%s:%s" % (a['uid'], Pass)
335 Line = Sanitize(Line) + "\n"
336 F.write("%s" % (Line))
338 # Oops, something unspeakable happened.
344 # Generate the sudo passwd file
345 def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
347 if sshcommand is None:
348 sshcommand = GitoliteSSHCommand
350 OldMask = os.umask(0022)
351 F = open(File + ".tmp", "w", 0600)
354 if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
356 if 'sshRSAAuthKey' not in a: continue
359 prefix = GitoliteSSHRestrictions
360 prefix = prefix.replace('@@COMMAND@@', sshcommand)
361 prefix = prefix.replace('@@USER@@', User)
362 for I in a["sshRSAAuthKey"]:
363 if I.startswith("allowed_hosts=") and ' ' in line:
364 if current_host is None:
366 machines, I = I.split('=', 1)[1].split(' ', 1)
367 if current_host not in machines.split(','):
368 continue # skip this key
370 if I.startswith('ssh-'):
371 line = "%s %s"%(prefix, I)
373 continue # do not allow keys with other restrictions that might conflict
374 line = Sanitize(line) + "\n"
377 for dn, attrs in hosts:
378 if 'sshRSAHostKey' not in attrs: continue
379 hostname = "host-" + attrs['hostname'][0]
380 prefix = GitoliteSSHRestrictions
381 prefix = prefix.replace('@@COMMAND@@', sshcommand)
382 prefix = prefix.replace('@@USER@@', hostname)
383 for I in attrs["sshRSAHostKey"]:
384 line = "%s %s"%(prefix, I)
385 line = Sanitize(line) + "\n"
388 # Oops, something unspeakable happened.
394 # Generate the shadow list
395 def GenSSHShadow(global_dir, accounts):
396 # Fetch all the users
400 if 'sshRSAAuthKey' not in a: continue
403 for I in a['sshRSAAuthKey']:
404 MultipleLine = "%s" % I
405 MultipleLine = Sanitize(MultipleLine)
406 contents.append(MultipleLine)
407 userkeys[a['uid']] = contents
410 # Generate the webPassword list
411 def GenWebPassword(accounts, File):
414 OldMask = os.umask(0077)
415 F = open(File, "w", 0600)
419 if 'webPassword' not in a: continue
420 if not a.pw_active(): continue
422 Pass = str(a['webPassword'])
423 Line = "%s:%s" % (a['uid'], Pass)
424 Line = Sanitize(Line) + "\n"
425 F.write("%s" % (Line))
431 # Generate the rtcPassword list
432 def GenRtcPassword(accounts, File):
435 OldMask = os.umask(0077)
436 F = open(File, "w", 0600)
440 if a.is_guest_account(): continue
441 if 'rtcPassword' not in a: continue
442 if not a.pw_active(): continue
444 Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
445 Line = Sanitize(Line) + "\n"
446 F.write("%s" % (Line))
452 # Generate the TOTP auth file
453 def GenTOTPSeed(accounts, File):
456 OldMask = os.umask(0077)
457 F = open(File, "w", 0600)
460 F.write("# Option User Prefix Seed\n")
462 if a.is_guest_account(): continue
463 if 'totpSeed' not in a: continue
464 if not a.pw_active(): continue
466 Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
467 Line = Sanitize(Line) + "\n"
468 F.write("%s" % (Line))
474 def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
475 OldMask = os.umask(0077)
476 tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
479 if f not in ssh_userkeys:
481 # If we're not exporting their primary group, don't export
484 if userlist[f] in grouprevmap.keys():
485 grname = grouprevmap[userlist[f]]
488 if int(userlist[f]) <= 100:
489 # In these cases, look it up in the normal way so we
490 # deal with cases where, for instance, users are in group
491 # users as their primary group.
492 grname = grp.getgrgid(userlist[f])[0]
497 print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f])
501 for line in ssh_userkeys[f]:
502 if line.startswith("allowed_hosts=") and ' ' in line:
503 machines, line = line.split('=', 1)[1].split(' ', 1)
504 if current_host not in machines.split(','):
505 continue # skip this key
508 continue # no keys for this host
509 contents = "\n".join(lines) + "\n"
511 to = tarfile.TarInfo(name=f)
512 # These will only be used where the username doesn't
513 # exist on the target system for some reason; hence,
514 # in those cases, the safest thing is for the file to
515 # be owned by root but group nobody. This deals with
516 # the bloody obscure case where the group fails to exist
517 # whilst the user does (in which case we want to avoid
518 # ending up with a file which is owned user:root to avoid
519 # a fairly obvious attack vector)
522 # Using the username / groupname fields avoids any need
523 # to give a shit^W^W^Wcare about the UIDoffset stuff.
527 to.mtime = int(time.time())
528 to.size = len(contents)
530 tf.addfile(to, StringIO(contents))
533 os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)
535 # add a list of groups to existing groups,
536 # including all subgroups thereof, recursively.
537 # basically this proceduces the transitive hull of the groups in
539 def addGroups(existingGroups, newGroups, uid, current_host):
540 for group in newGroups:
541 # if it's a <group>@host, split it and verify it's on the current host.
542 s = group.split('@', 1)
543 if len(s) == 2 and s[1] != current_host:
547 # let's see if we handled this group already
548 if group in existingGroups:
551 if not GroupIDMap.has_key(group):
552 print "Group", group, "does not exist but", uid, "is in it"
555 existingGroups.append(group)
557 if SubGroupMap.has_key(group):
558 addGroups(existingGroups, SubGroupMap[group], uid, current_host)
560 # Generate the group list
561 def GenGroup(accounts, File, current_host):
565 F = open(File + ".tdb.tmp", "w")
567 # Generate the GroupMap
571 GroupHasPrimaryMembers = {}
573 # Sort them into a list of groups having a set of users
575 GroupHasPrimaryMembers[ a['gidNumber'] ] = True
576 if 'supplementaryGid' not in a: continue
579 addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
581 GroupMap[g].append(a['uid'])
583 # Output the group file.
585 for x in GroupMap.keys():
586 if x not in GroupIDMap:
589 if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
592 grouprevmap[GroupIDMap[x]] = x
594 Line = "%s:x:%u:" % (x, GroupIDMap[x])
596 for I in GroupMap[x]:
597 Line = Line + ("%s%s" % (Comma, I))
599 Line = Sanitize(Line) + "\n"
600 F.write("0%u %s" % (J, Line))
601 F.write(".%s %s" % (x, Line))
602 F.write("=%u %s" % (GroupIDMap[x], Line))
605 # Oops, something unspeakable happened.
613 def CheckForward(accounts):
615 if 'emailForward' not in a: continue
619 # Do not allow people to try to buffer overflow busted parsers
620 if len(a['emailForward']) > 200: delete = True
621 # Check the forwarding address
622 elif EmailCheck.match(a['emailForward']) is None: delete = True
625 a.delete_mailforward()
627 # Generate the email forwarding list
628 def GenForward(accounts, File):
631 OldMask = os.umask(0022)
632 F = open(File + ".tmp", "w", 0644)
636 if 'emailForward' not in a: continue
637 Line = "%s: %s" % (a['uid'], a['emailForward'])
638 Line = Sanitize(Line) + "\n"
641 # Oops, something unspeakable happened.
647 def GenCDB(accounts, File, key):
648 prefix = ["/usr/bin/eatmydata"] if os.path.exists('/usr/bin/eatmydata') else []
649 # nothing else does the fsync stuff, so why do it here?
650 Fdb = subprocess.Popen(prefix + ["cdbmake", File, "%s.tmp" % File],
651 preexec_fn=lambda: os.umask(0022),
652 stdin=subprocess.PIPE)
654 # Write out the email address for each user
656 if key not in a: continue
659 Fdb.stdin.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
661 Fdb.stdin.write("\n")
665 raise Exception("cdbmake gave an error")
667 def GenDBM(accounts, File, key):
669 OldMask = os.umask(0022)
670 fn = os.path.join(File).encode('ascii', 'ignore')
677 Fdb = dbm.open(fn, "c")
680 # Write out the email address for each user
682 if key not in a: continue
689 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
690 os.remove(File + ".db")
692 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
693 os.rename (File + ".db", File)
695 # Generate the anon XEarth marker file
696 def GenMarkers(accounts, File):
699 F = open(File + ".tmp", "w")
701 # Write out the position for each user
703 if not ('latitude' in a and 'longitude' in a): continue
705 Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True))
706 Line = Sanitize(Line) + "\n"
711 # Oops, something unspeakable happened.
717 # Generate the debian-private subscription list
718 def GenPrivate(accounts, File):
721 F = open(File + ".tmp", "w")
723 # Write out the position for each user
725 if not a.is_active_user(): continue
726 if a.is_guest_account(): continue
727 if 'privateSub' not in a: continue
729 Line = "%s"%(a['privateSub'])
730 Line = Sanitize(Line) + "\n"
735 # Oops, something unspeakable happened.
741 # Generate a list of locked accounts
742 def GenDisabledAccounts(accounts, File):
745 F = open(File + ".tmp", "w")
746 disabled_accounts = []
748 # Fetch all the users
750 if a.pw_active(): continue
751 Line = "%s:%s" % (a['uid'], "Account is locked")
752 disabled_accounts.append(a)
753 F.write(Sanitize(Line) + "\n")
755 # Oops, something unspeakable happened.
760 return disabled_accounts
762 # Generate the list of local addresses that refuse all mail
763 def GenMailDisable(accounts, File):
766 F = open(File + ".tmp", "w")
769 if 'mailDisableMessage' not in a: continue
770 Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
771 Line = Sanitize(Line) + "\n"
774 # Oops, something unspeakable happened.
780 # Generate a list of uids that should have boolean affects applied
781 def GenMailBool(accounts, File, key):
784 F = open(File + ".tmp", "w")
787 if key not in a: continue
788 if not a[key] == 'TRUE': continue
789 Line = "%s"%(a['uid'])
790 Line = Sanitize(Line) + "\n"
793 # Oops, something unspeakable happened.
799 # Generate a list of hosts for RBL or whitelist purposes.
800 def GenMailList(accounts, File, key):
803 F = open(File + ".tmp", "w")
805 if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$')
806 else: validregex = re.compile('^[-\w.]+$')
809 if key not in a: continue
811 filtered = filter(lambda z: validregex.match(z), a[key])
812 if len(filtered) == 0: continue
813 if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered)
814 line = a['uid'] + ': ' + ' : '.join(filtered)
815 line = Sanitize(line) + "\n"
818 # Oops, something unspeakable happened.
824 def isRoleAccount(account):
825 return 'debianRoleAccount' in account['objectClass']
827 # Generate the DNS Zone file
828 def GenDNS(accounts, File):
831 F = open(File + ".tmp", "w")
833 # Fetch all the users
836 # Write out the zone file entry for each user
838 if 'dnsZoneEntry' not in a: continue
839 if not a.is_active_user() and not isRoleAccount(a): continue
840 if a.is_guest_account(): continue
843 F.write("; %s\n"%(a.email_address()))
844 for z in a["dnsZoneEntry"]:
845 Split = z.lower().split()
846 if Split[1].lower() == 'in':
847 Line = " ".join(Split) + "\n"
850 Host = Split[0] + DNSZone
851 if BSMTPCheck.match(Line) is not None:
852 F.write("; Has BSMTP\n")
854 # Write some identification information
855 if not RRs.has_key(Host):
856 if Split[2].lower() in ["a", "aaaa"]:
857 Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address())
858 for y in a["keyFingerPrint"]:
859 Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y))
863 Line = "; Err %s"%(str(Split))
868 F.write("; Errors:\n")
869 for line in str(e).split("\n"):
870 F.write("; %s\n"%(line))
873 # Oops, something unspeakable happened.
881 socket.inet_pton(socket.AF_INET6, i)
886 def ExtractDNSInfo(x):
887 hostname = GetAttr(x, "hostname")
891 TTLprefix="%s\t"%(x[1]["dnsTTL"][0])
894 if x[1].has_key("ipHostNumber"):
895 for I in x[1]["ipHostNumber"]:
897 DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I))
899 DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I))
903 ssh_hostnames = [ hostname ]
904 if x[1].has_key("sshfpHostname"):
905 ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ]
907 if 'sshRSAHostKey' in x[1]:
908 for I in x[1]["sshRSAHostKey"]:
910 key_prefix = Split[0]
911 key = base64.decodestring(Split[1])
914 # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
915 if key_prefix == 'ssh-rsa':
917 if key_prefix == 'ssh-dss':
919 if key_prefix == 'ssh-ed25519':
921 if Algorithm is None:
923 # and more from the registry
924 sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ]
926 fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ]
927 for h in ssh_hostnames:
928 for digest_codepoint, fingerprint in fingerprints:
929 DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))
931 if 'architecture' in x[1]:
932 Arch = GetAttr(x, "architecture")
934 if x[1].has_key("machine"):
935 Mach = " " + GetAttr(x, "machine")
936 DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))
938 if x[1].has_key("mXRecord"):
939 for I in x[1]["mXRecord"]:
941 for e in MX_remap[I]:
942 DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))
944 DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I))
948 # Generate the DNS records
949 def GenZoneRecords(host_attrs, File):
952 F = open(File + ".tmp", "w")
954 # Fetch all the hosts
956 if x[1].has_key("hostname") == 0:
959 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
962 for Line in ExtractDNSInfo(x):
965 # Oops, something unspeakable happened.
971 # Generate the BSMTP file
972 def GenBSMTP(accounts, File, HomePrefix):
975 F = open(File + ".tmp", "w")
977 # Write out the zone file entry for each user
979 if 'dnsZoneEntry' not in a: continue
980 if not a.is_active_user(): continue
983 for z in a["dnsZoneEntry"]:
984 Split = z.lower().split()
985 if Split[1].lower() == 'in':
986 for y in range(0, len(Split)):
989 Line = " ".join(Split) + "\n"
991 Host = Split[0] + DNSZone
992 if BSMTPCheck.match(Line) is not None:
993 F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
994 a['uid'], HomePrefix, a['uid'], Host))
997 F.write("; Errors\n")
1000 # Oops, something unspeakable happened.
1006 def HostToIP(Host, mapped=True):
1010 if Host[1].has_key("ipHostNumber"):
1011 for addr in Host[1]["ipHostNumber"]:
1012 IPAdresses.append(addr)
1013 if not is_ipv6_addr(addr) and mapped == "True":
1014 IPAdresses.append("::ffff:"+addr)
1018 # Generate the ssh known hosts file
1019 def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
1022 OldMask = os.umask(0022)
1023 F = open(File + ".tmp", "w", 0644)
1026 for x in host_attrs:
1027 if x[1].has_key("hostname") == 0 or \
1028 x[1].has_key("sshRSAHostKey") == 0:
1030 Host = GetAttr(x, "hostname")
1031 HostNames = [ Host ]
1032 if Host.endswith(HostDomain):
1033 HostNames.append(Host[:-(len(HostDomain) + 1)])
1035 # in the purpose field [[host|some other text]] (where some other text is optional)
1036 # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
1037 # file. But so that we don't have to add everything we link we can add an asterisk
1038 # and say [[*... to ignore it. In order to be able to add stuff to ssh without
1039 # http linking it we also support [[-hostname]] entries.
1040 for i in x[1].get("purpose", []):
1041 m = PurposeHostField.match(i)
1044 # we ignore [[*..]] entries
1045 if m.startswith('*'):
1047 if m.startswith('-'):
1051 if m.endswith(HostDomain):
1052 HostNames.append(m[:-(len(HostDomain) + 1)])
1054 for I in x[1]["sshRSAHostKey"]:
1055 if mode and mode == 'authorized_keys':
1057 if 'sshdistAuthKeysHost' in x[1]:
1058 hosts += x[1]['sshdistAuthKeysHost']
1059 clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host)
1060 clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand)
1061 Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I)
1063 Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I)
1064 Line = Sanitize(Line) + "\n"
1066 # Oops, something unspeakable happened.
1072 # Generate the debianhosts file (list of all IP addresses)
1073 def GenHosts(host_attrs, File):
1076 OldMask = os.umask(0022)
1077 F = open(File + ".tmp", "w", 0644)
1082 for x in host_attrs:
1084 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
1087 if 'ipHostNumber' not in x[1]:
1090 addrs = x[1]["ipHostNumber"]
1092 if addr not in seen:
1094 addr = Sanitize(addr) + "\n"
1097 # Oops, something unspeakable happened.
1103 def replaceTree(src, dst_basedir):
1104 bn = os.path.basename(src)
1105 dst = os.path.join(dst_basedir, bn)
1107 shutil.copytree(src, dst)
1109 def GenKeyrings(OutDir):
1111 if os.path.isdir(k):
1112 replaceTree(k, OutDir)
1114 shutil.copy(k, OutDir)
1117 def get_accounts(ldap_conn):
1118 # Fetch all the users
1119 passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\
1120 ["uid", "uidNumber", "gidNumber", "supplementaryGid",\
1121 "gecos", "loginShell", "userPassword", "shadowLastChange",\
1122 "shadowMin", "shadowMax", "shadowWarning", "shadowInactive",
1123 "shadowExpire", "emailForward", "latitude", "longitude",\
1124 "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\
1125 "keyFingerPrint", "privateSub", "mailDisableMessage",\
1126 "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
1127 "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
1128 "mailContentInspectionAction", "webPassword", "rtcPassword",\
1129 "bATVToken", "totpSeed", "mailDefaultOptions"])
1131 if passwd_attrs is None:
1132 raise UDEmptyList, "No Users"
1133 accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs)
1134 accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower()))
1138 def get_hosts(ldap_conn):
1139 # Fetch all the hosts
1140 HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
1141 ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
1142 "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
1145 if HostAttrs is None:
1146 raise UDEmptyList, "No Hosts"
1148 HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))
1153 def make_ldap_conn():
1154 # Connect to the ldap server
1156 # for testing purposes it's sometimes useful to pass username/password
1157 # via the environment
1158 if 'UD_CREDENTIALS' in os.environ:
1159 Pass = os.environ['UD_CREDENTIALS'].split()
1161 F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r")
1162 Pass = F.readline().strip().split(" ")
1164 l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1])
1170 def setup_group_maps(l):
1171 # Fetch all the groups
1174 attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
1175 ["gid", "gidNumber", "subGroup"])
1177 # Generate the subgroup_map and group_id_map
1179 if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":
1181 if x[1].has_key("gidNumber") == 0:
1183 group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
1184 if x[1].has_key("subGroup") != 0:
1185 subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])
1189 SubGroupMap = subgroup_map
1190 GroupIDMap = group_id_map
1192 def generate_all(global_dir, ldap_conn):
1193 accounts = get_accounts(ldap_conn)
1194 host_attrs = get_hosts(ldap_conn)
1197 # Generate global things
1198 accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
1200 accounts = filter(lambda x: not IsRetired(x), accounts)
1202 CheckForward(accounts)
1204 GenMailDisable(accounts, global_dir + "mail-disable")
1205 GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
1206 GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
1207 GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
1208 GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
1209 GenCDB(accounts, global_dir + "default-mail-options.cdb", 'mailDefaultOptions')
1210 GenDBM(accounts, global_dir + "default-mail-options.db", 'mailDefaultOptions')
1211 GenPrivate(accounts, global_dir + "debian-private")
1212 GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
1213 GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
1214 GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
1215 GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
1216 GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
1217 GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
1218 GenWebPassword(accounts, global_dir + "web-passwords")
1219 GenRtcPassword(accounts, global_dir + "rtc-passwords")
1220 GenTOTPSeed(accounts, global_dir + "users.oath")
1221 GenKeyrings(global_dir)
1224 GenForward(accounts, global_dir + "forward-alias")
1226 GenAllUsers(accounts, global_dir + 'all-accounts.json')
1227 accounts = filter(lambda a: a not in accounts_disabled, accounts)
1229 ssh_userkeys = GenSSHShadow(global_dir, accounts)
1230 GenMarkers(accounts, global_dir + "markers")
1231 GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
1232 GenHosts(host_attrs, global_dir + "debianhosts")
1234 GenDNS(accounts, global_dir + "dns-zone")
1235 GenZoneRecords(host_attrs, global_dir + "dns-sshfp")
1237 setup_group_maps(ldap_conn)
1239 for host in host_attrs:
1240 if "hostname" not in host[1]:
1242 generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
1244 def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
1245 current_host = host[1]['hostname'][0]
1246 OutDir = global_dir + current_host + '/'
1247 if not os.path.isdir(OutDir):
1250 # Get the group list and convert any named groups to numerics
1252 for groupname in AllowedGroupsPreload.strip().split(" "):
1253 GroupList[groupname] = True
1254 if 'allowedGroups' in host[1]:
1255 for groupname in host[1]['allowedGroups']:
1256 GroupList[groupname] = True
1257 for groupname in GroupList.keys():
1258 if groupname in GroupIDMap:
1259 GroupList[str(GroupIDMap[groupname])] = True
1262 if 'exportOptions' in host[1]:
1263 for extra in host[1]['exportOptions']:
1264 ExtraList[extra.upper()] = True
1267 accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)
1269 DoLink(global_dir, OutDir, "debianhosts")
1270 DoLink(global_dir, OutDir, "ssh_known_hosts")
1271 DoLink(global_dir, OutDir, "disabled-accounts")
1274 if 'NOPASSWD' in ExtraList:
1275 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*")
1277 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
1279 grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
1280 GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)
1282 # Now we know who we're allowing on the machine, export
1283 # the relevant ssh keys
1284 GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)
1286 if 'NOPASSWD' not in ExtraList:
1287 GenShadow(accounts, OutDir + "shadow")
1289 # Link in global things
1290 if 'NOMARKERS' not in ExtraList:
1291 DoLink(global_dir, OutDir, "markers")
1292 DoLink(global_dir, OutDir, "mail-forward.cdb")
1293 DoLink(global_dir, OutDir, "mail-forward.db")
1294 DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
1295 DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
1296 DoLink(global_dir, OutDir, "mail-disable")
1297 DoLink(global_dir, OutDir, "mail-greylist")
1298 DoLink(global_dir, OutDir, "mail-callout")
1299 DoLink(global_dir, OutDir, "mail-rbl")
1300 DoLink(global_dir, OutDir, "mail-rhsbl")
1301 DoLink(global_dir, OutDir, "mail-whitelist")
1302 DoLink(global_dir, OutDir, "all-accounts.json")
1303 DoLink(global_dir, Outdir, "default-mail-options.cdb")
1304 DoLink(global_dir, Outdir, "default-mail-options.db")
1305 GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
1306 GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
1307 GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
1308 GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')
1311 DoLink(global_dir, OutDir, "forward-alias")
1313 if 'DNS' in ExtraList:
1314 DoLink(global_dir, OutDir, "dns-zone")
1315 DoLink(global_dir, OutDir, "dns-sshfp")
1317 if 'AUTHKEYS' in ExtraList:
1318 DoLink(global_dir, OutDir, "authorized_keys")
1320 if 'BSMTP' in ExtraList:
1321 GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix)
1323 if 'PRIVATE' in ExtraList:
1324 DoLink(global_dir, OutDir, "debian-private")
1326 if 'GITOLITE' in ExtraList:
1327 GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)
1328 if 'exportOptions' in host[1]:
1329 for entry in host[1]['exportOptions']:
1330 v = entry.split('=',1)
1331 if v[0] != 'GITOLITE' or len(v) != 2: continue
1332 options = v[1].split(',')
1333 group = options.pop(0);
1334 gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
1335 if 'nohosts' not in options:
1336 gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
1341 if opt.startswith('sshcmd='):
1342 command = opt.split('=',1)[1]
1343 GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)
1345 if 'WEB-PASSWORDS' in ExtraList:
1346 DoLink(global_dir, OutDir, "web-passwords")
1348 if 'RTC-PASSWORDS' in ExtraList:
1349 DoLink(global_dir, OutDir, "rtc-passwords")
1351 if 'TOTP' in ExtraList:
1352 DoLink(global_dir, OutDir, "users.oath")
1354 if 'KEYRING' in ExtraList:
1356 bn = os.path.basename(k)
1357 if os.path.isdir(k):
1358 src = os.path.join(global_dir, bn)
1359 replaceTree(src, OutDir)
1361 DoLink(global_dir, OutDir, bn)
1365 bn = os.path.basename(k)
1366 target = os.path.join(OutDir, bn)
1367 if os.path.isdir(target):
1370 posix.remove(target)
1373 DoLink(global_dir, OutDir, "last_update.trace")
1376 def getLastLDAPChangeTime(l):
1377 mods = l.search_s('cn=log',
1378 ldap.SCOPE_ONELEVEL,
1379 '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))',
1384 # Sort the list by reqEnd
1385 sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0])
1386 # Take the last element in the array
1387 last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0]
1391 def getLastKeyringChangeTime():
1394 mt = os.path.getmtime(k)
1400 def getLastBuildTime(gdir):
1401 cache_last_ldap_mod = 0
1402 cache_last_unix_mod = 0
1406 fd = open(os.path.join(gdir, "last_update.trace"), "r")
1407 cache_last_mod=fd.read().split()
1409 cache_last_ldap_mod = cache_last_mod[0]
1410 cache_last_unix_mod = int(cache_last_mod[1])
1411 cache_last_run = int(cache_last_mod[2])
1412 except IndexError, ValueError:
1416 if e.errno == errno.ENOENT:
1421 return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
1423 def mq_notify(options, message):
1424 options.section = 'dsa-udgenerate'
1425 options.config = '/etc/dsa/pubsub.conf'
1427 config = Config(options)
1429 'rabbit_userid': config.username,
1430 'rabbit_password': config.password,
1431 'rabbit_virtual_host': config.vhost,
1432 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
1438 'timestamp': int(time.time())
1442 conn = Connection(conf=conf)
1443 conn.topic_send(config.topic,
1445 exchange_name=config.exchange,
1452 parser = optparse.OptionParser()
1453 parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
1454 help="Output directory.")
1455 parser.add_option("-f", "--force", dest="force", action="store_true",
1456 help="Force generation, even if no update to LDAP has happened.")
1458 (options, args) = parser.parse_args()
1463 if options.generatedir is not None:
1464 generate_dir = os.environ['UD_GENERATEDIR']
1465 elif 'UD_GENERATEDIR' in os.environ:
1466 generate_dir = os.environ['UD_GENERATEDIR']
1468 generate_dir = GenerateDir
1471 lockf = os.path.join(generate_dir, 'ud-generate.lock')
1472 lock = get_lock( lockf )
1474 sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
1477 l = make_ldap_conn()
1479 time_started = int(time.time())
1480 ldap_last_mod = getLastLDAPChangeTime(l)
1481 unix_last_mod = getLastKeyringChangeTime()
1482 cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)
1484 need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
1486 fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
1487 if need_update or options.force:
1488 msg = 'Update forced' if options.force else 'Update needed'
1489 generate_all(generate_dir, l)
1491 mq_notify(options, msg)
1492 last_run = int(time.time())
1493 fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
1498 if __name__ == "__main__":
1499 if 'UD_PROFILE' in os.environ:
1502 cProfile.run('ud_generate()', "udg_prof")
1503 p = pstats.Stats('udg_prof')
1504 ##p.sort_stats('time').print_stats()
1505 p.sort_stats('cumulative').print_stats()
1511 # vim:set shiftwidth=3: