X-Git-Url: https://git.adam-barratt.org.uk/?a=blobdiff_plain;f=ud-generate;h=9bad07a83a0354b561bd6b9f30c8767ad02a4e0f;hb=fe12f1a9d872cd6570ff1744b60629e8c26601f8;hp=02b2887119d409be367407b3cecf3efc98ef15df;hpb=6a15e53a56957d6c60acdda0352df406646d5d4e;p=mirror%2Fuserdir-ldap.git diff --git a/ud-generate b/ud-generate index 02b2887..9bad07a 100755 --- a/ud-generate +++ b/ud-generate @@ -6,7 +6,7 @@ # Copyright (c) 2003-2004 James Troup # Copyright (c) 2004-2005,7 Joey Schulze # Copyright (c) 2001-2007 Ryan Murray -# Copyright (c) 2008,2009,2010 Peter Palfrader +# Copyright (c) 2008,2009,2010,2011 Peter Palfrader # Copyright (c) 2008 Andreas Barth # Copyright (c) 2008 Mark Hymers # Copyright (c) 2008 Luk Claes @@ -28,7 +28,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import string, re, time, ldap, getopt, sys, os, pwd, posix, socket, base64, sha, shutil, errno, tarfile, grp +import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl from userdir_ldap import * from userdir_exceptions import * import UDLdap @@ -44,18 +44,18 @@ except ImportError: sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n") sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n") -global Allowed -global CurrentHost - if os.getuid() == 0: sys.stderr.write("You should probably not run ud-generate as root.\n") sys.exit(1) -DebianUsers = None -GroupIDMap = {} -SubGroupMap = {} -Allowed = None -CurrentHost = "" + +# +# GLOBAL STATE +# +GroupIDMap = None +SubGroupMap = None + + UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' @@ -67,6 +67,8 @@ IsDebianHost = re.compile(ConfModule.dns_hostmatch) isSSHFP = re.compile("^\s*IN\s+SSHFP") DNSZone = ".debian.net" Keyrings = ConfModule.sync_keyrings.split(":") +GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None) + def safe_makedirs(dir): try: @@ -86,6 +88,25 @@ def safe_rmtree(dir): else: raise e +def get_lock(fn, wait=5*60): + f = open(fn, "w") + sl = 0.1 + ends = time.time() + wait + + while True: + success = False + try: + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return f + except IOError: + pass + if time.time() >= ends: + return None + sl = min(sl*2, 10, ends - time.time()) + time.sleep(sl) + return None + + def Sanitize(Str): return Str.translate(string.maketrans("\n\r\t", "$$$")) @@ -129,23 +150,20 @@ def IsRetired(account): # return account['gidNumber'] == 800 # See if this user is in the group list -def IsInGroup(account): - if Allowed is None: - return True - +def IsInGroup(account, allowed, current_host): # See if the primary group is in the list - if str(account['gidNumber']) in Allowed: return True + if str(account['gidNumber']) in allowed: return True # Check the host based ACL - if account.is_allowed_by_hostacl(CurrentHost): return True + if account.is_allowed_by_hostacl(current_host): return True # See if there are supplementary groups if not 'supplementaryGid' in account: return False supgroups=[] - addGroups(supgroups, account['supplementaryGid'], account['uid']) + addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host) for g in supgroups: - if Allowed.has_key(g): + if allowed.has_key(g): return True return False @@ -180,8 +198,6 @@ def GenPasswd(accounts, File, HomePrefix, PwdMarker): userlist = {} i = 0 for a in accounts: - if not IsInGroup(a): continue - # Do not let people try to buffer overflow some busted passwd parser. if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue @@ -239,9 +255,6 @@ def GenShadow(accounts, File): i = 0 for a in accounts: - Pass = '*' - if not IsInGroup(a): continue - # If the account is locked, mark it as such in shadow # See Debian Bug #308229 for why we set it to 1 instead of 0 if not a.pw_active(): ShadowExpire = '1' @@ -268,7 +281,7 @@ def GenShadow(accounts, File): Done(File, None, F) # Generate the sudo passwd file -def GenShadowSudo(accounts, File, untrusted): +def GenShadowSudo(accounts, File, untrusted, current_host): F = None try: OldMask = os.umask(0077) @@ -277,8 +290,6 @@ def GenShadowSudo(accounts, File, untrusted): for a in accounts: Pass = '*' - if not IsInGroup(a): continue - if 'sudoPassword' in a: for entry in a['sudoPassword']: Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry) @@ -292,7 +303,7 @@ def GenShadowSudo(accounts, File, untrusted): if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass): continue for_all = hosts == "*" - for_this_host = CurrentHost in hosts.split(',') + for_this_host = current_host in hosts.split(',') if not (for_all or for_this_host): continue # ignore * passwords for untrusted hosts, but copy host specific passwords @@ -314,48 +325,98 @@ def GenShadowSudo(accounts, File, untrusted): raise Done(File, F, None) +# Generate the sudo passwd file +def GenSSHGitolite(accounts, File): + F = None + try: + OldMask = os.umask(0022) + F = open(File + ".tmp", "w", 0600) + os.umask(OldMask) + + if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "": + for a in accounts: + if not 'sshRSAAuthKey' in a: continue + + User = a['uid'] + prefix = GitoliteSSHRestrictions.replace('@@USER@@', User) + for I in a["sshRSAAuthKey"]: + if I.startswith('ssh-'): + line = "%s %s"%(prefix, I) + else: + line = "%s,%s"%(prefix, I) + line = Sanitize(line) + "\n" + F.write(line) + + # Oops, something unspeakable happened. + except: + Die(File, F, None) + raise + Done(File, F, None) + # Generate the shadow list -def GenSSHShadow(accounts): +def GenSSHShadow(global_dir, accounts): # Fetch all the users - userfiles = [] - - safe_rmtree(os.path.join(GlobalDir, 'userkeys')) - safe_makedirs(os.path.join(GlobalDir, 'userkeys')) + userkeys = {} for a in accounts: if not 'sshRSAAuthKey' in a: continue - F = None - try: - OldMask = os.umask(0077) - File = os.path.join(GlobalDir, 'userkeys', a['uid']) - F = open(File + ".tmp", "w", 0600) - os.umask(OldMask) + contents = [] + for I in a['sshRSAAuthKey']: + MultipleLine = "%s" % I + MultipleLine = Sanitize(MultipleLine) + contents.append(MultipleLine) + userkeys[a['uid']] = contents + return userkeys + +# Generate the webPassword list +def GenWebPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) - for I in a['sshRSAAuthKey']: - MultipleLine = "%s" % I - MultipleLine = Sanitize(MultipleLine) + "\n" - F.write(MultipleLine) + for a in accounts: + if not 'webPassword' in a: continue + if not a.pw_active(): continue - Done(File, F, None) - userfiles.append(os.path.basename(File)) + Pass = str(a['webPassword']) + Line = "%s:%s" % (a['uid'], Pass) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) - # Oops, something unspeakable happened. - except IOError: - Die(File, F, None) - # As neither masterFileName nor masterFile are defined at any point - # this will raise a NameError. - Die(masterFileName, masterFile, None) - raise + except: + Die(File, None, F) + raise - return userfiles +# Generate the voipPassword list +def GenVoipPassword(accounts, File): + F = None + try: + OldMask = os.umask(0077) + F = open(File, "w", 0600) + os.umask(OldMask) + + for a in accounts: + if not 'voipPassword' in a: continue + if not a.pw_active(): continue -def GenSSHtarballs(userlist, SSHFiles, grouprevmap, target): + Pass = str(a['voipPassword']) + Line = "\n \n \n \n" % (a['uid'], Pass) + Line = Sanitize(Line) + "\n" + F.write("%s" % (Line)) + + except: + Die(File, None, F) + raise + +def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host): OldMask = os.umask(0077) - tf = tarfile.open(name=os.path.join(GlobalDir, 'ssh-keys-%s.tar.gz' % CurrentHost), mode='w:gz') + tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz') os.umask(OldMask) - for f in userlist.keys(): - if f not in SSHFiles: + for f in userlist: + if f not in ssh_userkeys: continue # If we're not exporting their primary group, don't export # the key and warn @@ -373,10 +434,21 @@ def GenSSHtarballs(userlist, SSHFiles, grouprevmap, target): pass if grname is None: - print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, CurrentHost, userlist[f]) + print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f]) continue - to = tf.gettarinfo(os.path.join(GlobalDir, 'userkeys', f), f) + lines = [] + for line in ssh_userkeys[f]: + if line.startswith("allowed_hosts=") and ' ' in line: + machines, line = line.split('=', 1)[1].split(' ', 1) + if current_host not in machines.split(','): + continue # skip this key + lines.append(line) + if not lines: + continue # no keys for this host + contents = "\n".join(lines) + "\n" + + to = tarfile.TarInfo(name=f) # These will only be used where the username doesn't # exist on the target system for some reason; hence, # in those cases, the safest thing is for the file to @@ -392,33 +464,23 @@ def GenSSHtarballs(userlist, SSHFiles, grouprevmap, target): to.uname = f to.gname = grname to.mode = 0400 - - contents = file(os.path.join(GlobalDir, 'userkeys', f)).read() - lines = [] - for line in contents.splitlines(): - if line.startswith("allowed_hosts=") and ' ' in line: - machines, line = line.split('=', 1)[1].split(' ', 1) - if CurrentHost not in machines.split(','): - continue # skip this key - lines.append(line) - if not lines: - continue # no keys for this host - contents = "\n".join(lines) + "\n" + to.mtime = int(time.time()) to.size = len(contents) + tf.addfile(to, StringIO(contents)) tf.close() - os.rename(os.path.join(GlobalDir, 'ssh-keys-%s.tar.gz' % CurrentHost), target) + os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target) # add a list of groups to existing groups, # including all subgroups thereof, recursively. # basically this proceduces the transitive hull of the groups in # addgroups. -def addGroups(existingGroups, newGroups, uid): +def addGroups(existingGroups, newGroups, uid, current_host): for group in newGroups: # if it's a @host, split it and verify it's on the current host. s = group.split('@', 1) - if len(s) == 2 and s[1] != CurrentHost: + if len(s) == 2 and s[1] != current_host: continue group = s[0] @@ -433,10 +495,10 @@ def addGroups(existingGroups, newGroups, uid): existingGroups.append(group) if SubGroupMap.has_key(group): - addGroups(existingGroups, SubGroupMap[group], uid) + addGroups(existingGroups, SubGroupMap[group], uid, current_host) # Generate the group list -def GenGroup(accounts, File): +def GenGroup(accounts, File, current_host): grouprevmap = {} F = None try: @@ -444,25 +506,24 @@ def GenGroup(accounts, File): # Generate the GroupMap GroupMap = {} - for x in GroupIDMap.keys(): + for x in GroupIDMap: GroupMap[x] = [] GroupHasPrimaryMembers = {} # Sort them into a list of groups having a set of users for a in accounts: GroupHasPrimaryMembers[ a['gidNumber'] ] = True - if not IsInGroup(a): continue if not 'supplementaryGid' in a: continue supgroups=[] - addGroups(supgroups, a['supplementaryGid'], a['uid']) + addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host) for g in supgroups: GroupMap[g].append(a['uid']) # Output the group file. J = 0 for x in GroupMap.keys(): - if GroupIDMap.has_key(x) == 0: + if not x in GroupIDMap: continue if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers: @@ -493,12 +554,10 @@ def CheckForward(accounts): for a in accounts: if not 'emailForward' in a: continue - delete = False - if not IsInGroup(a): delete = True # Do not allow people to try to buffer overflow busted parsers - elif len(a['emailForward']) > 200: delete = True + if len(a['emailForward']) > 200: delete = True # Check the forwarding address elif EmailCheck.match(a['emailForward']) is None: delete = True @@ -529,7 +588,9 @@ def GenCDB(accounts, File, key): Fdb = None try: OldMask = os.umask(0022) - Fdb = os.popen("cdbmake %s %s.tmp"%(File, File), "w") + # nothing else does the fsync stuff, so why do it here? + prefix = "/usr/bin/eatmydata " if os.path.exists('/usr/bin/eatmydata') else '' + Fdb = os.popen("%scdbmake %s %s.tmp"%(prefix, File, File), "w") os.umask(OldMask) # Write out the email address for each user @@ -697,16 +758,13 @@ def GenDNS(accounts, File): for z in a["dnsZoneEntry"]: Split = z.lower().split() if Split[1].lower() == 'in': - for y in range(0, len(Split)): - if Split[y] == "$": - Split[y] = "\n\t" Line = " ".join(Split) + "\n" F.write(Line) - + Host = Split[0] + DNSZone if BSMTPCheck.match(Line) != None: F.write("; Has BSMTP\n") - + # Write some identification information if not RRs.has_key(Host): if Split[2].lower() in ["a", "aaaa"]: @@ -718,7 +776,7 @@ def GenDNS(accounts, File): else: Line = "; Err %s"%(str(Split)) F.write(Line) - + F.write("\n") except Exception, e: F.write("; Errors:\n") @@ -757,7 +815,7 @@ def ExtractDNSInfo(x): Algorithm = 2 if Algorithm == None: continue - Fingerprint = sha.new(base64.decodestring(Split[1])).hexdigest() + Fingerprint = hashlib.new('sha1', base64.decodestring(Split[1])).hexdigest() DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint)) if 'architecture' in x[1]: @@ -774,15 +832,13 @@ def ExtractDNSInfo(x): return DNSInfo # Generate the DNS records -def GenZoneRecords(File): +def GenZoneRecords(host_attrs, File): F = None try: F = open(File + ".tmp", "w") # Fetch all the hosts - global HostAttrs - - for x in HostAttrs: + for x in host_attrs: if x[1].has_key("hostname") == 0: continue @@ -877,16 +933,14 @@ def HostToIP(Host, mapped=True): return IPAdresses # Generate the ssh known hosts file -def GenSSHKnown(File, mode=None): +def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None): F = None try: OldMask = os.umask(0022) F = open(File + ".tmp", "w", 0644) os.umask(OldMask) - global HostAttrs - - for x in HostAttrs: + for x in host_attrs: if x[1].has_key("hostname") == 0 or \ x[1].has_key("sshRSAHostKey") == 0: continue @@ -919,7 +973,9 @@ def GenSSHKnown(File, mode=None): hosts = HostToIP(x) if 'sshdistAuthKeysHost' in x[1]: hosts += x[1]['sshdistAuthKeysHost'] - Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (Host, ",".join(hosts), I) + clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host) + clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand) + Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I) else: Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I) Line = Sanitize(Line) + "\n" @@ -931,7 +987,7 @@ def GenSSHKnown(File, mode=None): Done(File, F, None) # Generate the debianhosts file (list of all IP addresses) -def GenHosts(File): +def GenHosts(host_attrs, File): F = None try: OldMask = os.umask(0022) @@ -940,9 +996,7 @@ def GenHosts(File): seen = set() - global HostAttrs - - for x in HostAttrs: + for x in host_attrs: if IsDebianHost.match(GetAttr(x, "hostname")) is None: continue @@ -963,110 +1017,147 @@ def GenHosts(File): raise Done(File, F, None) +def replaceTree(src, dst_basedir): + bn = os.path.basename(src) + dst = os.path.join(dst_basedir, bn) + safe_rmtree(dst) + shutil.copytree(src, dst) + def GenKeyrings(OutDir): for k in Keyrings: - shutil.copy(k, OutDir) - -# Connect to the ldap server -l = connectLDAP() -# for testing purposes it's sometimes useful to pass username/password -# via the environment -if 'UD_CREDENTIALS' in os.environ: - Pass = os.environ['UD_CREDENTIALS'].split() -else: - F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r") - Pass = F.readline().strip().split(" ") - F.close() -l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1]) - -# Fetch all the groups -GroupIDMap = {} -Attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\ - ["gid", "gidNumber", "subGroup"]) - -# Generate the SubGroupMap and GroupIDMap -for x in Attrs: - if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled": - continue - if x[1].has_key("gidNumber") == 0: - continue - GroupIDMap[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]) - if x[1].has_key("subGroup") != 0: - SubGroupMap.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"]) - -# Fetch all the users -passwd_attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0)))",\ - ["uid", "uidNumber", "gidNumber", "supplementaryGid",\ - "gecos", "loginShell", "userPassword", "shadowLastChange",\ - "shadowMin", "shadowMax", "shadowWarning", "shadowInactive", - "shadowExpire", "emailForward", "latitude", "longitude",\ - "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\ - "keyFingerPrint", "privateSub", "mailDisableMessage",\ - "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ - "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ - "mailContentInspectionAction"]) - -if passwd_attrs is None: - raise UDEmptyList, "No Users" -accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs) -accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower())) - -# Fetch all the hosts -HostAttrs = l.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\ - ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\ - "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"]) - -if HostAttrs == None: - raise UDEmptyList, "No Hosts" - -HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower())) - -# override globaldir for testing -if 'UD_GENERATEDIR' in os.environ: - GenerateDir = os.environ['UD_GENERATEDIR'] - -# Generate global things -GlobalDir = GenerateDir + "/" -accounts_disabled = GenDisabledAccounts(accounts, GlobalDir + "disabled-accounts") - -accounts = filter(lambda x: not IsRetired(x), accounts) -#accounts_DDs = filter(lambda x: IsGidDebian(x), accounts) - -CheckForward(accounts) - -GenMailDisable(accounts, GlobalDir + "mail-disable") -GenCDB(accounts, GlobalDir + "mail-forward.cdb", 'emailForward') -GenCDB(accounts, GlobalDir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') -GenPrivate(accounts, GlobalDir + "debian-private") -GenSSHKnown(GlobalDir+"authorized_keys", 'authorized_keys') -GenMailBool(accounts, GlobalDir + "mail-greylist", "mailGreylisting") -GenMailBool(accounts, GlobalDir + "mail-callout", "mailCallout") -GenMailList(accounts, GlobalDir + "mail-rbl", "mailRBL") -GenMailList(accounts, GlobalDir + "mail-rhsbl", "mailRHSBL") -GenMailList(accounts, GlobalDir + "mail-whitelist", "mailWhitelist") -GenKeyrings(GlobalDir) - -# Compatibility. -GenForward(accounts, GlobalDir + "forward-alias") - -GenAllUsers(accounts, GlobalDir + 'all-accounts.json') -accounts = filter(lambda a: not a in accounts_disabled, accounts) - -SSHFiles = GenSSHShadow(accounts) -GenMarkers(accounts, GlobalDir + "markers") -GenSSHKnown(GlobalDir + "ssh_known_hosts") -GenHosts(GlobalDir + "debianhosts") - -for host in HostAttrs: - if not "hostname" in host[1]: - continue - - CurrentHost = host[1]['hostname'][0] - OutDir = GlobalDir + CurrentHost + '/' - try: + if os.path.isdir(k): + replaceTree(k, OutDir) + else: + shutil.copy(k, OutDir) + + +def get_accounts(ldap_conn): + # Fetch all the users + passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\ + ["uid", "uidNumber", "gidNumber", "supplementaryGid",\ + "gecos", "loginShell", "userPassword", "shadowLastChange",\ + "shadowMin", "shadowMax", "shadowWarning", "shadowInactive", + "shadowExpire", "emailForward", "latitude", "longitude",\ + "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\ + "keyFingerPrint", "privateSub", "mailDisableMessage",\ + "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\ + "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\ + "mailContentInspectionAction", "webPassword", "voipPassword"]) + + if passwd_attrs is None: + raise UDEmptyList, "No Users" + accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs) + accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower())) + + return accounts + +def get_hosts(ldap_conn): + # Fetch all the hosts + HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\ + ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\ + "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"]) + + if HostAttrs == None: + raise UDEmptyList, "No Hosts" + + HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower())) + + return HostAttrs + + +def make_ldap_conn(): + # Connect to the ldap server + l = connectLDAP() + # for testing purposes it's sometimes useful to pass username/password + # via the environment + if 'UD_CREDENTIALS' in os.environ: + Pass = os.environ['UD_CREDENTIALS'].split() + else: + F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r") + Pass = F.readline().strip().split(" ") + F.close() + l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1]) + + return l + + + +def setup_group_maps(l): + # Fetch all the groups + group_id_map = {} + subgroup_map = {} + attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\ + ["gid", "gidNumber", "subGroup"]) + + # Generate the subgroup_map and group_id_map + for x in attrs: + if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled": + continue + if x[1].has_key("gidNumber") == 0: + continue + group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0]) + if x[1].has_key("subGroup") != 0: + subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"]) + + global SubGroupMap + global GroupIDMap + SubGroupMap = subgroup_map + GroupIDMap = group_id_map + +def generate_all(global_dir, ldap_conn): + accounts = get_accounts(ldap_conn) + host_attrs = get_hosts(ldap_conn) + + global_dir += '/' + # Generate global things + accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts") + + accounts = filter(lambda x: not IsRetired(x), accounts) + #accounts_DDs = filter(lambda x: IsGidDebian(x), accounts) + + CheckForward(accounts) + + GenMailDisable(accounts, global_dir + "mail-disable") + GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward') + GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction') + GenPrivate(accounts, global_dir + "debian-private") + GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock') + GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting") + GenMailBool(accounts, global_dir + "mail-callout", "mailCallout") + GenMailList(accounts, global_dir + "mail-rbl", "mailRBL") + GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL") + GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist") + GenWebPassword(accounts, global_dir + "web-passwords") + GenVoipPassword(accounts, global_dir + "voip-passwords") + GenKeyrings(global_dir) + + # Compatibility. + GenForward(accounts, global_dir + "forward-alias") + + GenAllUsers(accounts, global_dir + 'all-accounts.json') + accounts = filter(lambda a: not a in accounts_disabled, accounts) + + ssh_userkeys = GenSSHShadow(global_dir, accounts) + GenMarkers(accounts, global_dir + "markers") + GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts") + GenHosts(host_attrs, global_dir + "debianhosts") + GenSSHGitolite(accounts, global_dir + "ssh-gitolite") + + GenDNS(accounts, global_dir + "dns-zone") + GenZoneRecords(host_attrs, global_dir + "dns-sshfp") + + setup_group_maps(ldap_conn) + + for host in host_attrs: + if not "hostname" in host[1]: + continue + generate_host(host, global_dir, accounts, ssh_userkeys) + +def generate_host(host, global_dir, accounts, ssh_userkeys): + current_host = host[1]['hostname'][0] + OutDir = global_dir + current_host + '/' + if not os.path.isdir(OutDir): os.mkdir(OutDir) - except: - pass # Get the group list and convert any named groups to numerics GroupList = {} @@ -1084,13 +1175,12 @@ for host in HostAttrs: for extra in host[1]['exportOptions']: ExtraList[extra.upper()] = True - Allowed = GroupList - if Allowed == {}: - Allowed = None + if GroupList != {}: + accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), accounts) - DoLink(GlobalDir, OutDir, "debianhosts") - DoLink(GlobalDir, OutDir, "ssh_known_hosts") - DoLink(GlobalDir, OutDir, "disabled-accounts") + DoLink(global_dir, OutDir, "debianhosts") + DoLink(global_dir, OutDir, "ssh_known_hosts") + DoLink(global_dir, OutDir, "disabled-accounts") sys.stdout.flush() if 'NOPASSWD' in ExtraList: @@ -1098,57 +1188,165 @@ for host in HostAttrs: else: userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x") sys.stdout.flush() - grouprevmap = GenGroup(accounts, OutDir + "group") - GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList)) + grouprevmap = GenGroup(accounts, OutDir + "group", current_host) + GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host) # Now we know who we're allowing on the machine, export # the relevant ssh keys - GenSSHtarballs(userlist, SSHFiles, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz')) + GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host) if not 'NOPASSWD' in ExtraList: GenShadow(accounts, OutDir + "shadow") # Link in global things if not 'NOMARKERS' in ExtraList: - DoLink(GlobalDir, OutDir, "markers") - DoLink(GlobalDir, OutDir, "mail-forward.cdb") - DoLink(GlobalDir, OutDir, "mail-contentinspectionaction.cdb") - DoLink(GlobalDir, OutDir, "mail-disable") - DoLink(GlobalDir, OutDir, "mail-greylist") - DoLink(GlobalDir, OutDir, "mail-callout") - DoLink(GlobalDir, OutDir, "mail-rbl") - DoLink(GlobalDir, OutDir, "mail-rhsbl") - DoLink(GlobalDir, OutDir, "mail-whitelist") - DoLink(GlobalDir, OutDir, "all-accounts.json") - GenCDB(filter(lambda x: IsInGroup(x), accounts), OutDir + "user-forward.cdb", 'emailForward') - GenCDB(filter(lambda x: IsInGroup(x), accounts), OutDir + "batv-tokens.cdb", 'bATVToken') - GenCDB(filter(lambda x: IsInGroup(x), accounts), OutDir + "default-mail-options.cdb", 'mailDefaultOptions') + DoLink(global_dir, OutDir, "markers") + DoLink(global_dir, OutDir, "mail-forward.cdb") + DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb") + DoLink(global_dir, OutDir, "mail-disable") + DoLink(global_dir, OutDir, "mail-greylist") + DoLink(global_dir, OutDir, "mail-callout") + DoLink(global_dir, OutDir, "mail-rbl") + DoLink(global_dir, OutDir, "mail-rhsbl") + DoLink(global_dir, OutDir, "mail-whitelist") + DoLink(global_dir, OutDir, "all-accounts.json") + GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward') + GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken') + GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions') # Compatibility. - DoLink(GlobalDir, OutDir, "forward-alias") + DoLink(global_dir, OutDir, "forward-alias") if 'DNS' in ExtraList: - GenDNS(accounts, OutDir + "dns-zone") - GenZoneRecords(OutDir + "dns-sshfp") + DoLink(global_dir, OutDir, "dns-zone") + DoLink(global_dir, OutDir, "dns-sshfp") if 'AUTHKEYS' in ExtraList: - DoLink(GlobalDir, OutDir, "authorized_keys") + DoLink(global_dir, OutDir, "authorized_keys") if 'BSMTP' in ExtraList: GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix) if 'PRIVATE' in ExtraList: - DoLink(GlobalDir, OutDir, "debian-private") + DoLink(global_dir, OutDir, "debian-private") + + if 'GITOLITE' in ExtraList: + DoLink(global_dir, OutDir, "ssh-gitolite") + + if 'WEB-PASSWORDS' in ExtraList: + DoLink(global_dir, OutDir, "web-passwords") if 'KEYRING' in ExtraList: for k in Keyrings: - DoLink(GlobalDir, OutDir, os.path.basename(k)) + bn = os.path.basename(k) + if os.path.isdir(k): + src = os.path.join(global_dir, bn) + replaceTree(src, OutDir) + else: + DoLink(global_dir, OutDir, bn) else: for k in Keyrings: - try: - posix.remove(OutDir + os.path.basename(k)) + try: + bn = os.path.basename(k) + target = os.path.join(OutDir, bn) + if os.path.isdir(target): + safe_rmtree(dst) + else: + posix.remove(target) except: pass + DoLink(global_dir, OutDir, "last_update.trace") + + +def getLastLDAPChangeTime(l): + mods = l.search_s('cn=log', + ldap.SCOPE_ONELEVEL, + '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))', + ['reqEnd']) + + last = 0 + + # Sort the list by reqEnd + sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0]) + # Take the last element in the array + last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0] + + return last + +def getLastBuildTime(gdir): + cache_last_mod = 0 + + try: + fd = open(os.path.join(gdir, "last_update.trace"), "r") + cache_last_mod=fd.read().split() + try: + cache_last_mod = cache_last_mod[0] + except IndexError: + pass + fd.close() + except IOError, e: + if e.errno == errno.ENOENT: + pass + else: + raise e + + return cache_last_mod + + +def ud_generate(): + parser = optparse.OptionParser() + parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR", + help="Output directory.") + parser.add_option("-f", "--force", dest="force", action="store_true", + help="Force generation, even if not update to LDAP has happened.") + + (options, args) = parser.parse_args() + if len(args) > 0: + parser.print_help() + sys.exit(1) + + if options.generatedir is not None: + generate_dir = os.environ['UD_GENERATEDIR'] + elif 'UD_GENERATEDIR' in os.environ: + generate_dir = os.environ['UD_GENERATEDIR'] + else: + generate_dir = GenerateDir + + + lockf = os.path.join(generate_dir, 'ud-generate.lock') + lock = get_lock( lockf ) + if lock is None: + sys.stderr.write("Could not acquire lock %s.\n"%(lockf)) + sys.exit(1) + + l = make_ldap_conn() + + ldap_last_mod = getLastLDAPChangeTime(l) + cache_last_mod = getLastBuildTime(generate_dir) + need_update = ldap_last_mod > cache_last_mod + + if not options.force and not need_update: + fd = open(os.path.join(generate_dir, "last_update.trace"), "w") + fd.write("%s\n%s\n" % (ldap_last_mod, int(time.time()))) + fd.close() + sys.exit(0) + + tracefd = open(os.path.join(generate_dir, "last_update.trace"), "w") + generate_all(generate_dir, l) + tracefd.write("%s\n%s\n" % (ldap_last_mod, int(time.time()))) + tracefd.close() + + +if __name__ == "__main__": + if 'UD_PROFILE' in os.environ: + import cProfile + import pstats + cProfile.run('ud_generate()', "udg_prof") + p = pstats.Stats('udg_prof') + ##p.sort_stats('time').print_stats() + p.sort_stats('cumulative').print_stats() + else: + ud_generate() # vim:set et: # vim:set ts=3: