3 # Generates passwd, shadow and group files from the ldap directory.
5 # Copyright (c) 2000-2001 Jason Gunthorpe <jgg@debian.org>
6 # Copyright (c) 2003-2004 James Troup <troup@debian.org>
7 # Copyright (c) 2004-2005,7 Joey Schulze <joey@infodrom.org>
8 # Copyright (c) 2001-2007 Ryan Murray <rmurray@debian.org>
9 # Copyright (c) 2008,2009,2010,2011 Peter Palfrader <peter@palfrader.org>
10 # Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
11 # Copyright (c) 2008 Mark Hymers <mhy@debian.org>
12 # Copyright (c) 2008 Luk Claes <luk@debian.org>
13 # Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>
14 # Copyright (c) 2009 Stephen Gran <steve@lobefin.net>
15 # Copyright (c) 2010 Helmut Grohne <helmut@subdivi.de>
17 # This program is free software; you can redistribute it and/or modify
18 # it under the terms of the GNU General Public License as published by
19 # the Free Software Foundation; either version 2 of the License, or
20 # (at your option) any later version.
22 # This program is distributed in the hope that it will be useful,
23 # but WITHOUT ANY WARRANTY; without even the implied warranty of
24 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 # GNU General Public License for more details.
27 # You should have received a copy of the GNU General Public License
28 # along with this program; if not, write to the Free Software
29 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
31 from dsa_mq.connection import Connection
32 from dsa_mq.config import Config
34 import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
35 from userdir_ldap import *
36 from userdir_exceptions import *
38 from xml.etree.ElementTree import Element, SubElement, Comment
39 from xml.etree import ElementTree
40 from xml.dom import minidom
42 from cStringIO import StringIO
44 from StringIO import StringIO
46 import simplejson as json
49 if not '__author__' in json.__dict__:
50 sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n")
51 sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n")
54 sys.stderr.write("You should probably not run ud-generate as root.\n")
66 UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
69 EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
70 BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL)
71 PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
72 IsDebianHost = re.compile(ConfModule.dns_hostmatch)
73 isSSHFP = re.compile("^\s*IN\s+SSHFP")
74 DNSZone = ".debian.net"
75 Keyrings = ConfModule.sync_keyrings.split(":")
76 GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
77 GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
78 MX_remap = json.loads(ConfModule.MX_remap)
81 """Return a pretty-printed XML string for the Element.
83 rough_string = ElementTree.tostring(elem, 'utf-8')
84 reparsed = minidom.parseString(rough_string)
85 return reparsed.toprettyxml(indent=" ")
87 def safe_makedirs(dir):
91 if e.errno == errno.EEXIST:
100 if e.errno == errno.ENOENT:
105 def get_lock(fn, wait=5*60):
108 ends = time.time() + wait
113 fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
117 if time.time() >= ends:
119 sl = min(sl*2, 10, ends - time.time())
125 return Str.translate(string.maketrans("\n\r\t", "$$$"))
127 def DoLink(From, To, File):
129 posix.remove(To + File)
132 posix.link(From + File, To + File)
134 def IsRetired(account):
136 Looks for accountStatus in the LDAP record and tries to
137 match it against one of the known retired statuses
140 status = account['accountStatus']
142 line = status.split()
145 if status == "inactive":
148 elif status == "memorial":
151 elif status == "retiring":
152 # We'll give them a few extra days over what we said
153 age = 6 * 31 * 24 * 60 * 60
155 return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age
163 #def IsGidDebian(account):
164 # return account['gidNumber'] == 800
166 # See if this user is in the group list
167 def IsInGroup(account, allowed, current_host):
168 # See if the primary group is in the list
169 if str(account['gidNumber']) in allowed: return True
171 # Check the host based ACL
172 if account.is_allowed_by_hostacl(current_host): return True
174 # See if there are supplementary groups
175 if not 'supplementaryGid' in account: return False
178 addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
184 def Die(File, F, Fdb):
190 os.remove(File + ".tmp")
194 os.remove(File + ".tdb.tmp")
198 def Done(File, F, Fdb):
201 os.rename(File + ".tmp", File)
204 os.rename(File + ".tdb.tmp", File + ".tdb")
206 # Generate the password list
207 def GenPasswd(accounts, File, HomePrefix, PwdMarker):
210 F = open(File + ".tdb.tmp", "w")
215 # Do not let people try to buffer overflow some busted passwd parser.
216 if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue
218 userlist[a['uid']] = a['gidNumber']
219 line = "%s:%s:%d:%d:%s:%s%s:%s" % (
225 HomePrefix, a['uid'],
227 line = Sanitize(line) + "\n"
228 F.write("0%u %s" % (i, line))
229 F.write(".%s %s" % (a['uid'], line))
230 F.write("=%d %s" % (a['uidNumber'], line))
233 # Oops, something unspeakable happened.
239 # Return the list of users so we know which keys to export
242 def GenAllUsers(accounts, file):
245 OldMask = os.umask(0022)
246 f = open(file + ".tmp", "w", 0644)
251 all.append( { 'uid': a['uid'],
252 'uidNumber': a['uidNumber'],
253 'active': a.pw_active() and a.shadow_active() } )
256 # Oops, something unspeakable happened.
262 # Generate the shadow list
263 def GenShadow(accounts, File):
266 OldMask = os.umask(0077)
267 F = open(File + ".tdb.tmp", "w", 0600)
272 # If the account is locked, mark it as such in shadow
273 # See Debian Bug #308229 for why we set it to 1 instead of 0
274 if not a.pw_active(): ShadowExpire = '1'
275 elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire'])
276 else: ShadowExpire = ''
279 values.append(a['uid'])
280 values.append(a.get_password())
281 for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive':
282 if key in a: values.append(a[key])
283 else: values.append('')
284 values.append(ShadowExpire)
285 line = ':'.join(values)+':'
286 line = Sanitize(line) + "\n"
287 F.write("0%u %s" % (i, line))
288 F.write(".%s %s" % (a['uid'], line))
291 # Oops, something unspeakable happened.
297 # Generate the sudo passwd file
298 def GenShadowSudo(accounts, File, untrusted, current_host):
301 OldMask = os.umask(0077)
302 F = open(File + ".tmp", "w", 0600)
307 if 'sudoPassword' in a:
308 for entry in a['sudoPassword']:
309 Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry)
312 uuid = Match.group(1)
313 status = Match.group(2)
314 hosts = Match.group(3)
315 cryptedpass = Match.group(4)
317 if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
319 for_all = hosts == "*"
320 for_this_host = current_host in hosts.split(',')
321 if not (for_all or for_this_host):
323 # ignore * passwords for untrusted hosts, but copy host specific passwords
324 if for_all and untrusted:
327 if for_this_host: # this makes sure we take a per-host entry over the for-all entry
332 Line = "%s:%s" % (a['uid'], Pass)
333 Line = Sanitize(Line) + "\n"
334 F.write("%s" % (Line))
336 # Oops, something unspeakable happened.
342 # Generate the sudo passwd file
343 def GenSSHGitolite(accounts, hosts, File):
346 OldMask = os.umask(0022)
347 F = open(File + ".tmp", "w", 0600)
350 if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
352 if not 'sshRSAAuthKey' in a: continue
355 prefix = GitoliteSSHRestrictions.replace('@@USER@@', User)
356 for I in a["sshRSAAuthKey"]:
357 if I.startswith('ssh-'):
358 line = "%s %s"%(prefix, I)
360 line = "%s,%s"%(prefix, I)
361 line = Sanitize(line) + "\n"
364 for dn, attrs in hosts:
365 if not 'sshRSAHostKey' in attrs: continue
366 hostname = "host-" + attrs['hostname'][0]
367 prefix = GitoliteSSHRestrictions.replace('@@USER@@', hostname)
368 for I in attrs["sshRSAHostKey"]:
369 line = "%s %s"%(prefix, I)
370 line = Sanitize(line) + "\n"
373 # Oops, something unspeakable happened.
379 # Generate the shadow list
380 def GenSSHShadow(global_dir, accounts):
381 # Fetch all the users
385 if not 'sshRSAAuthKey' in a: continue
388 for I in a['sshRSAAuthKey']:
389 MultipleLine = "%s" % I
390 MultipleLine = Sanitize(MultipleLine)
391 contents.append(MultipleLine)
392 userkeys[a['uid']] = contents
395 # Generate the webPassword list
396 def GenWebPassword(accounts, File):
399 OldMask = os.umask(0077)
400 F = open(File, "w", 0600)
404 if not 'webPassword' in a: continue
405 if not a.pw_active(): continue
407 Pass = str(a['webPassword'])
408 Line = "%s:%s" % (a['uid'], Pass)
409 Line = Sanitize(Line) + "\n"
410 F.write("%s" % (Line))
416 # Generate the rtcPassword list
417 def GenRtcPassword(accounts, File):
420 OldMask = os.umask(0077)
421 F = open(File, "w", 0600)
425 if not 'rtcPassword' in a: continue
426 if not a.pw_active(): continue
428 Line = "%s@debian.org:%s:rtc.debian.org:AUTHORIZED" % (a['uid'], str(a['rtcPassword']))
429 Line = Sanitize(Line) + "\n"
430 F.write("%s" % (Line))
436 def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
437 OldMask = os.umask(0077)
438 tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
441 if f not in ssh_userkeys:
443 # If we're not exporting their primary group, don't export
446 if userlist[f] in grouprevmap.keys():
447 grname = grouprevmap[userlist[f]]
450 if int(userlist[f]) <= 100:
451 # In these cases, look it up in the normal way so we
452 # deal with cases where, for instance, users are in group
453 # users as their primary group.
454 grname = grp.getgrgid(userlist[f])[0]
459 print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f])
463 for line in ssh_userkeys[f]:
464 if line.startswith("allowed_hosts=") and ' ' in line:
465 machines, line = line.split('=', 1)[1].split(' ', 1)
466 if current_host not in machines.split(','):
467 continue # skip this key
470 continue # no keys for this host
471 contents = "\n".join(lines) + "\n"
473 to = tarfile.TarInfo(name=f)
474 # These will only be used where the username doesn't
475 # exist on the target system for some reason; hence,
476 # in those cases, the safest thing is for the file to
477 # be owned by root but group nobody. This deals with
478 # the bloody obscure case where the group fails to exist
479 # whilst the user does (in which case we want to avoid
480 # ending up with a file which is owned user:root to avoid
481 # a fairly obvious attack vector)
484 # Using the username / groupname fields avoids any need
485 # to give a shit^W^W^Wcare about the UIDoffset stuff.
489 to.mtime = int(time.time())
490 to.size = len(contents)
492 tf.addfile(to, StringIO(contents))
495 os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)
497 # add a list of groups to existing groups,
498 # including all subgroups thereof, recursively.
499 # basically this proceduces the transitive hull of the groups in
501 def addGroups(existingGroups, newGroups, uid, current_host):
502 for group in newGroups:
503 # if it's a <group>@host, split it and verify it's on the current host.
504 s = group.split('@', 1)
505 if len(s) == 2 and s[1] != current_host:
509 # let's see if we handled this group already
510 if group in existingGroups:
513 if not GroupIDMap.has_key(group):
514 print "Group", group, "does not exist but", uid, "is in it"
517 existingGroups.append(group)
519 if SubGroupMap.has_key(group):
520 addGroups(existingGroups, SubGroupMap[group], uid, current_host)
522 # Generate the group list
523 def GenGroup(accounts, File, current_host):
527 F = open(File + ".tdb.tmp", "w")
529 # Generate the GroupMap
533 GroupHasPrimaryMembers = {}
535 # Sort them into a list of groups having a set of users
537 GroupHasPrimaryMembers[ a['gidNumber'] ] = True
538 if not 'supplementaryGid' in a: continue
541 addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
543 GroupMap[g].append(a['uid'])
545 # Output the group file.
547 for x in GroupMap.keys():
548 if not x in GroupIDMap:
551 if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
554 grouprevmap[GroupIDMap[x]] = x
556 Line = "%s:x:%u:" % (x, GroupIDMap[x])
558 for I in GroupMap[x]:
559 Line = Line + ("%s%s" % (Comma, I))
561 Line = Sanitize(Line) + "\n"
562 F.write("0%u %s" % (J, Line))
563 F.write(".%s %s" % (x, Line))
564 F.write("=%u %s" % (GroupIDMap[x], Line))
567 # Oops, something unspeakable happened.
575 def CheckForward(accounts):
577 if not 'emailForward' in a: continue
581 # Do not allow people to try to buffer overflow busted parsers
582 if len(a['emailForward']) > 200: delete = True
583 # Check the forwarding address
584 elif EmailCheck.match(a['emailForward']) is None: delete = True
587 a.delete_mailforward()
589 # Generate the email forwarding list
590 def GenForward(accounts, File):
593 OldMask = os.umask(0022)
594 F = open(File + ".tmp", "w", 0644)
598 if not 'emailForward' in a: continue
599 Line = "%s: %s" % (a['uid'], a['emailForward'])
600 Line = Sanitize(Line) + "\n"
603 # Oops, something unspeakable happened.
609 def GenCDB(accounts, File, key):
612 OldMask = os.umask(0022)
613 # nothing else does the fsync stuff, so why do it here?
614 prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else ''
615 Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w")
618 # Write out the email address for each user
620 if not key in a: continue
623 Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
626 # Oops, something unspeakable happened.
630 if Fdb.close() != None:
631 raise "cdbmake gave an error"
633 def GenDBM(accounts, File, key):
635 OldMask = os.umask(0022)
636 fn = os.path.join(File).encode('ascii', 'ignore')
643 Fdb = dbm.open(fn, "c")
646 # Write out the email address for each user
648 if not key in a: continue
655 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
656 os.remove(File + ".db")
658 # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
659 os.rename (File + ".db", File)
661 # Generate the anon XEarth marker file
662 def GenMarkers(accounts, File):
665 F = open(File + ".tmp", "w")
667 # Write out the position for each user
669 if not ('latitude' in a and 'longitude' in a): continue
671 Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True))
672 Line = Sanitize(Line) + "\n"
677 # Oops, something unspeakable happened.
683 # Generate the debian-private subscription list
684 def GenPrivate(accounts, File):
687 F = open(File + ".tmp", "w")
689 # Write out the position for each user
691 if not a.is_active_user(): continue
692 if a.is_guest_account(): continue
693 if not 'privateSub' in a: continue
695 Line = "%s"%(a['privateSub'])
696 Line = Sanitize(Line) + "\n"
701 # Oops, something unspeakable happened.
707 # Generate a list of locked accounts
708 def GenDisabledAccounts(accounts, File):
711 F = open(File + ".tmp", "w")
712 disabled_accounts = []
714 # Fetch all the users
716 if a.pw_active(): continue
717 Line = "%s:%s" % (a['uid'], "Account is locked")
718 disabled_accounts.append(a)
719 F.write(Sanitize(Line) + "\n")
721 # Oops, something unspeakable happened.
726 return disabled_accounts
728 # Generate the list of local addresses that refuse all mail
729 def GenMailDisable(accounts, File):
732 F = open(File + ".tmp", "w")
735 if not 'mailDisableMessage' in a: continue
736 Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
737 Line = Sanitize(Line) + "\n"
740 # Oops, something unspeakable happened.
746 # Generate a list of uids that should have boolean affects applied
747 def GenMailBool(accounts, File, key):
750 F = open(File + ".tmp", "w")
753 if not key in a: continue
754 if not a[key] == 'TRUE': continue
755 Line = "%s"%(a['uid'])
756 Line = Sanitize(Line) + "\n"
759 # Oops, something unspeakable happened.
765 # Generate a list of hosts for RBL or whitelist purposes.
766 def GenMailList(accounts, File, key):
769 F = open(File + ".tmp", "w")
771 if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$')
772 else: validregex = re.compile('^[-\w.]+$')
775 if not key in a: continue
777 filtered = filter(lambda z: validregex.match(z), a[key])
778 if len(filtered) == 0: continue
779 if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered)
780 line = a['uid'] + ': ' + ' : '.join(filtered)
781 line = Sanitize(line) + "\n"
784 # Oops, something unspeakable happened.
790 def isRoleAccount(account):
791 return 'debianRoleAccount' in account['objectClass']
793 # Generate the DNS Zone file
794 def GenDNS(accounts, File):
797 F = open(File + ".tmp", "w")
799 # Fetch all the users
802 # Write out the zone file entry for each user
804 if not 'dnsZoneEntry' in a: continue
805 if not a.is_active_user() and not isRoleAccount(a): continue
806 if a.is_guest_account(): continue
809 F.write("; %s\n"%(a.email_address()))
810 for z in a["dnsZoneEntry"]:
811 Split = z.lower().split()
812 if Split[1].lower() == 'in':
813 Line = " ".join(Split) + "\n"
816 Host = Split[0] + DNSZone
817 if BSMTPCheck.match(Line) != None:
818 F.write("; Has BSMTP\n")
820 # Write some identification information
821 if not RRs.has_key(Host):
822 if Split[2].lower() in ["a", "aaaa"]:
823 Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address())
824 for y in a["keyFingerPrint"]:
825 Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y))
829 Line = "; Err %s"%(str(Split))
834 F.write("; Errors:\n")
835 for line in str(e).split("\n"):
836 F.write("; %s\n"%(line))
839 # Oops, something unspeakable happened.
847 socket.inet_pton(socket.AF_INET6, i)
852 def ExtractDNSInfo(x):
856 TTLprefix="%s\t"%(x[1]["dnsTTL"][0])
859 if x[1].has_key("ipHostNumber"):
860 for I in x[1]["ipHostNumber"]:
862 DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I))
864 DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I))
868 if 'sshRSAHostKey' in x[1]:
869 for I in x[1]["sshRSAHostKey"]:
871 if Split[0] == 'ssh-rsa':
873 if Split[0] == 'ssh-dss':
875 if Algorithm == None:
877 Fingerprint = hashlib.new('sha1', base64.decodestring(Split[1])).hexdigest()
878 DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint))
880 if 'architecture' in x[1]:
881 Arch = GetAttr(x, "architecture")
883 if x[1].has_key("machine"):
884 Mach = " " + GetAttr(x, "machine")
885 DNSInfo.append("%sIN\tHINFO\t\"%s%s\" \"%s\"" % (TTLprefix, Arch, Mach, "Debian GNU/Linux"))
887 if x[1].has_key("mXRecord"):
888 for I in x[1]["mXRecord"]:
890 for e in MX_remap[I]:
891 DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, e))
893 DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I))
897 # Generate the DNS records
898 def GenZoneRecords(host_attrs, File):
901 F = open(File + ".tmp", "w")
903 # Fetch all the hosts
905 if x[1].has_key("hostname") == 0:
908 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
911 DNSInfo = ExtractDNSInfo(x)
915 Line = "%s.\t%s" % (GetAttr(x, "hostname"), Line)
918 Line = "\t\t\t%s" % (Line)
922 # this would write sshfp lines for services on machines
923 # but we can't yet, since some are cnames and we'll make
924 # an invalid zonefile
926 # for i in x[1].get("purpose", []):
927 # m = PurposeHostField.match(i)
930 # # we ignore [[*..]] entries
931 # if m.startswith('*'):
933 # if m.startswith('-'):
936 # if not m.endswith(HostDomain):
938 # if not m.endswith('.'):
940 # for Line in DNSInfo:
941 # if isSSHFP.match(Line):
942 # Line = "%s\t%s" % (m, Line)
943 # F.write(Line + "\n")
945 # Oops, something unspeakable happened.
951 # Generate the BSMTP file
952 def GenBSMTP(accounts, File, HomePrefix):
955 F = open(File + ".tmp", "w")
957 # Write out the zone file entry for each user
959 if not 'dnsZoneEntry' in a: continue
960 if not a.is_active_user(): continue
963 for z in a["dnsZoneEntry"]:
964 Split = z.lower().split()
965 if Split[1].lower() == 'in':
966 for y in range(0, len(Split)):
969 Line = " ".join(Split) + "\n"
971 Host = Split[0] + DNSZone
972 if BSMTPCheck.match(Line) != None:
973 F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
974 a['uid'], HomePrefix, a['uid'], Host))
977 F.write("; Errors\n")
980 # Oops, something unspeakable happened.
986 def HostToIP(Host, mapped=True):
990 if Host[1].has_key("ipHostNumber"):
991 for addr in Host[1]["ipHostNumber"]:
992 IPAdresses.append(addr)
993 if not is_ipv6_addr(addr) and mapped == "True":
994 IPAdresses.append("::ffff:"+addr)
998 # Generate the ssh known hosts file
999 def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
1002 OldMask = os.umask(0022)
1003 F = open(File + ".tmp", "w", 0644)
1006 for x in host_attrs:
1007 if x[1].has_key("hostname") == 0 or \
1008 x[1].has_key("sshRSAHostKey") == 0:
1010 Host = GetAttr(x, "hostname")
1011 HostNames = [ Host ]
1012 if Host.endswith(HostDomain):
1013 HostNames.append(Host[:-(len(HostDomain) + 1)])
1015 # in the purpose field [[host|some other text]] (where some other text is optional)
1016 # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
1017 # file. But so that we don't have to add everything we link we can add an asterisk
1018 # and say [[*... to ignore it. In order to be able to add stuff to ssh without
1019 # http linking it we also support [[-hostname]] entries.
1020 for i in x[1].get("purpose", []):
1021 m = PurposeHostField.match(i)
1024 # we ignore [[*..]] entries
1025 if m.startswith('*'):
1027 if m.startswith('-'):
1031 if m.endswith(HostDomain):
1032 HostNames.append(m[:-(len(HostDomain) + 1)])
1034 for I in x[1]["sshRSAHostKey"]:
1035 if mode and mode == 'authorized_keys':
1037 if 'sshdistAuthKeysHost' in x[1]:
1038 hosts += x[1]['sshdistAuthKeysHost']
1039 clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host)
1040 clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand)
1041 Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I)
1043 Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I)
1044 Line = Sanitize(Line) + "\n"
1046 # Oops, something unspeakable happened.
1052 # Generate the debianhosts file (list of all IP addresses)
1053 def GenHosts(host_attrs, File):
1056 OldMask = os.umask(0022)
1057 F = open(File + ".tmp", "w", 0644)
1062 for x in host_attrs:
1064 if IsDebianHost.match(GetAttr(x, "hostname")) is None:
1067 if not 'ipHostNumber' in x[1]:
1070 addrs = x[1]["ipHostNumber"]
1072 if addr not in seen:
1074 addr = Sanitize(addr) + "\n"
1077 # Oops, something unspeakable happened.
1083 def replaceTree(src, dst_basedir):
1084 bn = os.path.basename(src)
1085 dst = os.path.join(dst_basedir, bn)
1087 shutil.copytree(src, dst)
1089 def GenKeyrings(OutDir):
1091 if os.path.isdir(k):
1092 replaceTree(k, OutDir)
1094 shutil.copy(k, OutDir)
1097 def get_accounts(ldap_conn):
1098 # Fetch all the users
1099 passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\
1100 ["uid", "uidNumber", "gidNumber", "supplementaryGid",\
1101 "gecos", "loginShell", "userPassword", "shadowLastChange",\
1102 "shadowMin", "shadowMax", "shadowWarning", "shadowInactive",
1103 "shadowExpire", "emailForward", "latitude", "longitude",\
1104 "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\
1105 "keyFingerPrint", "privateSub", "mailDisableMessage",\
1106 "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
1107 "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
1108 "mailContentInspectionAction", "webPassword", "rtcPassword",\
1111 if passwd_attrs is None:
1112 raise UDEmptyList, "No Users"
1113 accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs)
1114 accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower()))
1118 def get_hosts(ldap_conn):
1119 # Fetch all the hosts
1120 HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
1121 ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
1122 "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"])
1124 if HostAttrs == None:
1125 raise UDEmptyList, "No Hosts"
1127 HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))
1132 def make_ldap_conn():
1133 # Connect to the ldap server
1135 # for testing purposes it's sometimes useful to pass username/password
1136 # via the environment
1137 if 'UD_CREDENTIALS' in os.environ:
1138 Pass = os.environ['UD_CREDENTIALS'].split()
1140 F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r")
1141 Pass = F.readline().strip().split(" ")
1143 l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1])
1149 def setup_group_maps(l):
1150 # Fetch all the groups
1153 attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
1154 ["gid", "gidNumber", "subGroup"])
1156 # Generate the subgroup_map and group_id_map
1158 if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":
1160 if x[1].has_key("gidNumber") == 0:
1162 group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
1163 if x[1].has_key("subGroup") != 0:
1164 subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])
1168 SubGroupMap = subgroup_map
1169 GroupIDMap = group_id_map
1171 def generate_all(global_dir, ldap_conn):
1172 accounts = get_accounts(ldap_conn)
1173 host_attrs = get_hosts(ldap_conn)
1176 # Generate global things
1177 accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
1179 accounts = filter(lambda x: not IsRetired(x), accounts)
1180 #accounts_DDs = filter(lambda x: IsGidDebian(x), accounts)
1182 CheckForward(accounts)
1184 GenMailDisable(accounts, global_dir + "mail-disable")
1185 GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
1186 GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
1187 GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
1188 GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
1189 GenPrivate(accounts, global_dir + "debian-private")
1190 GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
1191 GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
1192 GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
1193 GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
1194 GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
1195 GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
1196 GenWebPassword(accounts, global_dir + "web-passwords")
1197 GenRtcPassword(accounts, global_dir + "rtc-passwords")
1198 GenKeyrings(global_dir)
1201 GenForward(accounts, global_dir + "forward-alias")
1203 GenAllUsers(accounts, global_dir + 'all-accounts.json')
1204 accounts = filter(lambda a: not a in accounts_disabled, accounts)
1206 ssh_userkeys = GenSSHShadow(global_dir, accounts)
1207 GenMarkers(accounts, global_dir + "markers")
1208 GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
1209 GenHosts(host_attrs, global_dir + "debianhosts")
1210 GenSSHGitolite(accounts, host_attrs, global_dir + "ssh-gitolite")
1212 GenDNS(accounts, global_dir + "dns-zone")
1213 GenZoneRecords(host_attrs, global_dir + "dns-sshfp")
1215 setup_group_maps(ldap_conn)
1217 for host in host_attrs:
1218 if not "hostname" in host[1]:
1220 generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
1222 def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
1223 current_host = host[1]['hostname'][0]
1224 OutDir = global_dir + current_host + '/'
1225 if not os.path.isdir(OutDir):
1228 # Get the group list and convert any named groups to numerics
1230 for groupname in AllowedGroupsPreload.strip().split(" "):
1231 GroupList[groupname] = True
1232 if 'allowedGroups' in host[1]:
1233 for groupname in host[1]['allowedGroups']:
1234 GroupList[groupname] = True
1235 for groupname in GroupList.keys():
1236 if groupname in GroupIDMap:
1237 GroupList[str(GroupIDMap[groupname])] = True
1240 if 'exportOptions' in host[1]:
1241 for extra in host[1]['exportOptions']:
1242 ExtraList[extra.upper()] = True
1245 accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)
1247 DoLink(global_dir, OutDir, "debianhosts")
1248 DoLink(global_dir, OutDir, "ssh_known_hosts")
1249 DoLink(global_dir, OutDir, "disabled-accounts")
1252 if 'NOPASSWD' in ExtraList:
1253 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*")
1255 userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
1257 grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
1258 GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)
1260 # Now we know who we're allowing on the machine, export
1261 # the relevant ssh keys
1262 GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)
1264 if not 'NOPASSWD' in ExtraList:
1265 GenShadow(accounts, OutDir + "shadow")
1267 # Link in global things
1268 if not 'NOMARKERS' in ExtraList:
1269 DoLink(global_dir, OutDir, "markers")
1270 DoLink(global_dir, OutDir, "mail-forward.cdb")
1271 DoLink(global_dir, OutDir, "mail-forward.db")
1272 DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
1273 DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
1274 DoLink(global_dir, OutDir, "mail-disable")
1275 DoLink(global_dir, OutDir, "mail-greylist")
1276 DoLink(global_dir, OutDir, "mail-callout")
1277 DoLink(global_dir, OutDir, "mail-rbl")
1278 DoLink(global_dir, OutDir, "mail-rhsbl")
1279 DoLink(global_dir, OutDir, "mail-whitelist")
1280 DoLink(global_dir, OutDir, "all-accounts.json")
1281 GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
1282 GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
1283 GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
1284 GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')
1285 GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions')
1286 GenDBM(accounts, OutDir + "default-mail-options.db", 'mailDefaultOptions')
1289 DoLink(global_dir, OutDir, "forward-alias")
1291 if 'DNS' in ExtraList:
1292 DoLink(global_dir, OutDir, "dns-zone")
1293 DoLink(global_dir, OutDir, "dns-sshfp")
1295 if 'AUTHKEYS' in ExtraList:
1296 DoLink(global_dir, OutDir, "authorized_keys")
1298 if 'BSMTP' in ExtraList:
1299 GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix)
1301 if 'PRIVATE' in ExtraList:
1302 DoLink(global_dir, OutDir, "debian-private")
1304 if 'GITOLITE' in ExtraList:
1305 DoLink(global_dir, OutDir, "ssh-gitolite")
1306 if 'exportOptions' in host[1]:
1307 for entry in host[1]['exportOptions']:
1308 v = entry.split('=',1)
1309 if v[0] != 'GITOLITE' or len(v) != 2: continue
1310 gitolite_accounts = filter(lambda x: IsInGroup(x, [v[1]], current_host), all_accounts)
1311 gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
1312 GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(v[1],))
1314 if 'WEB-PASSWORDS' in ExtraList:
1315 DoLink(global_dir, OutDir, "web-passwords")
1317 if 'RTC-PASSWORDS' in ExtraList:
1318 DoLink(global_dir, OutDir, "rtc-passwords")
1320 if 'KEYRING' in ExtraList:
1322 bn = os.path.basename(k)
1323 if os.path.isdir(k):
1324 src = os.path.join(global_dir, bn)
1325 replaceTree(src, OutDir)
1327 DoLink(global_dir, OutDir, bn)
1331 bn = os.path.basename(k)
1332 target = os.path.join(OutDir, bn)
1333 if os.path.isdir(target):
1336 posix.remove(target)
1339 DoLink(global_dir, OutDir, "last_update.trace")
1342 def getLastLDAPChangeTime(l):
1343 mods = l.search_s('cn=log',
1344 ldap.SCOPE_ONELEVEL,
1345 '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))',
1350 # Sort the list by reqEnd
1351 sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0])
1352 # Take the last element in the array
1353 last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0]
1357 def getLastKeyringChangeTime():
1360 mt = os.path.getmtime(k)
1366 def getLastBuildTime(gdir):
1367 cache_last_ldap_mod = 0
1368 cache_last_unix_mod = 0
1372 fd = open(os.path.join(gdir, "last_update.trace"), "r")
1373 cache_last_mod=fd.read().split()
1375 cache_last_ldap_mod = cache_last_mod[0]
1376 cache_last_unix_mod = int(cache_last_mod[1])
1377 cache_last_run = int(cache_last_mod[2])
1378 except IndexError, ValueError:
1382 if e.errno == errno.ENOENT:
1387 return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
1389 def mq_notify(options, message):
1390 options.section = 'dsa-udgenerate'
1391 options.config = '/etc/dsa/pubsub.conf'
1393 config = Config(options)
1395 'rabbit_userid': config.username,
1396 'rabbit_password': config.password,
1397 'rabbit_virtual_host': config.vhost,
1398 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
1404 'timestamp': int(time.time())
1408 conn = Connection(conf=conf)
1409 conn.topic_send(config.topic,
1411 exchange_name=config.exchange,
1418 parser = optparse.OptionParser()
1419 parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
1420 help="Output directory.")
1421 parser.add_option("-f", "--force", dest="force", action="store_true",
1422 help="Force generation, even if no update to LDAP has happened.")
1424 (options, args) = parser.parse_args()
1429 if options.generatedir is not None:
1430 generate_dir = os.environ['UD_GENERATEDIR']
1431 elif 'UD_GENERATEDIR' in os.environ:
1432 generate_dir = os.environ['UD_GENERATEDIR']
1434 generate_dir = GenerateDir
1437 lockf = os.path.join(generate_dir, 'ud-generate.lock')
1438 lock = get_lock( lockf )
1440 sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
1443 l = make_ldap_conn()
1445 time_started = int(time.time())
1446 ldap_last_mod = getLastLDAPChangeTime(l)
1447 unix_last_mod = getLastKeyringChangeTime()
1448 cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)
1450 need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
1452 fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
1453 if need_update or options.force:
1454 msg = 'Update forced' if options.force else 'Update needed'
1455 generate_all(generate_dir, l)
1456 mq_notify(options, msg)
1457 last_run = int(time.time())
1458 fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
1463 if __name__ == "__main__":
1464 if 'UD_PROFILE' in os.environ:
1467 cProfile.run('ud_generate()', "udg_prof")
1468 p = pstats.Stats('udg_prof')
1469 ##p.sort_stats('time').print_stats()
1470 p.sort_stats('cumulative').print_stats()
1476 # vim:set shiftwidth=3: