# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp
-import lockfile
+from dsa_mq.connection import Connection
+from dsa_mq.config import Config
+
+import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
from userdir_ldap import *
from userdir_exceptions import *
import UDLdap
+from xml.etree.ElementTree import Element, SubElement, Comment
+from xml.etree import ElementTree
+from xml.dom import minidom
try:
from cStringIO import StringIO
except ImportError:
import simplejson as json
except ImportError:
import json
- if not '__author__' in json.__dict__:
+ if '__author__' not in json.__dict__:
sys.stderr.write("Warning: This is probably the wrong json module. We want python 2.6's json\n")
sys.stderr.write("module, or simplejson on pytyon 2.5. Let's see if/how stuff blows up.\n")
#
# GLOBAL STATE
#
-GroupIDMap = {}
-SubGroupMap = {}
-CurrentHost = ""
+GroupIDMap = None
+SubGroupMap = None
UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+MAX_UD_AGE = 3600*24
-EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)?$")
+EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL)
PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
-IsV6Addr = re.compile("^[a-fA-F0-9:]+$")
IsDebianHost = re.compile(ConfModule.dns_hostmatch)
isSSHFP = re.compile("^\s*IN\s+SSHFP")
DNSZone = ".debian.net"
Keyrings = ConfModule.sync_keyrings.split(":")
GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
+GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
+GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
+MX_remap = json.loads(ConfModule.MX_remap)
+use_mq = getattr(ConfModule, "use_mq", True)
+
+rtc_realm = getattr(ConfModule, "rtc_realm", None)
+rtc_append = getattr(ConfModule, "rtc_append", None)
+def prettify(elem):
+ """Return a pretty-printed XML string for the Element.
+ """
+ rough_string = ElementTree.tostring(elem, 'utf-8')
+ reparsed = minidom.parseString(rough_string)
+ return reparsed.toprettyxml(indent=" ")
def safe_makedirs(dir):
try:
else:
raise e
-def get_lock(fn, wait=5*60, max_age=3600*6):
- try:
- stat = os.stat(fn + '.lock')
- if stat.st_mtime < time.time() - max_age:
- sys.stderr.write("Removing stale lock %s"%(fn + '.lock'))
- os.unlink(fn + '.lock')
- except OSError, error:
- if error.errno == errno.ENOENT:
- pass
- else:
- raise
-
- lock = lockfile.FileLock(fn)
- try:
- lock.acquire(timeout=wait)
- except lockfile.LockTimeout:
- return None
+def get_lock(fn, wait=5*60):
+ f = open(fn, "w")
+ sl = 0.1
+ ends = time.time() + wait
- return lock
+ while True:
+ success = False
+ try:
+ fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ return f
+ except IOError:
+ pass
+ if time.time() >= ends:
+ return None
+ sl = min(sl*2, 10, ends - time.time())
+ time.sleep(sl)
+ return None
def Sanitize(Str):
return False
-#def IsGidDebian(account):
-# return account['gidNumber'] == 800
-
# See if this user is in the group list
-def IsInGroup(account, allowed):
+def IsInGroup(account, allowed, current_host):
# See if the primary group is in the list
if str(account['gidNumber']) in allowed: return True
# Check the host based ACL
- if account.is_allowed_by_hostacl(CurrentHost): return True
+ if account.is_allowed_by_hostacl(current_host): return True
# See if there are supplementary groups
- if not 'supplementaryGid' in account: return False
+ if 'supplementaryGid' not in account: return False
supgroups=[]
- addGroups(supgroups, account['supplementaryGid'], account['uid'])
+ addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
for g in supgroups:
- if allowed.has_key(g):
+ if g in allowed:
return True
return False
def Die(File, F, Fdb):
- if F != None:
+ if F is not None:
F.close()
- if Fdb != None:
+ if Fdb is not None:
Fdb.close()
try:
os.remove(File + ".tmp")
pass
def Done(File, F, Fdb):
- if F != None:
+ if F is not None:
F.close()
os.rename(File + ".tmp", File)
- if Fdb != None:
+ if Fdb is not None:
Fdb.close()
os.rename(File + ".tdb.tmp", File + ".tdb")
Done(File, None, F)
# Generate the sudo passwd file
-def GenShadowSudo(accounts, File, untrusted):
+def GenShadowSudo(accounts, File, untrusted, current_host):
F = None
try:
OldMask = os.umask(0077)
Pass = '*'
if 'sudoPassword' in a:
for entry in a['sudoPassword']:
- Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*]+) ([^ ]+)$').match(entry)
- if Match == None:
+ Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
+ if Match is None:
continue
uuid = Match.group(1)
status = Match.group(2)
if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
continue
for_all = hosts == "*"
- for_this_host = CurrentHost in hosts.split(',')
+ for_this_host = current_host in hosts.split(',')
if not (for_all or for_this_host):
continue
# ignore * passwords for untrusted hosts, but copy host specific passwords
Done(File, F, None)
# Generate the sudo passwd file
-def GenSSHGitolite(accounts, File):
+def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
F = None
+ if sshcommand is None:
+ sshcommand = GitoliteSSHCommand
try:
OldMask = os.umask(0022)
F = open(File + ".tmp", "w", 0600)
if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
for a in accounts:
- if not 'sshRSAAuthKey' in a: continue
+ if 'sshRSAAuthKey' not in a: continue
User = a['uid']
- prefix = GitoliteSSHRestrictions.replace('@@USER@@', User)
+ prefix = GitoliteSSHRestrictions
+ prefix = prefix.replace('@@COMMAND@@', sshcommand)
+ prefix = prefix.replace('@@USER@@', User)
for I in a["sshRSAAuthKey"]:
+ if I.startswith("allowed_hosts=") and ' ' in line:
+ if current_host is None:
+ continue
+ machines, I = I.split('=', 1)[1].split(' ', 1)
+ if current_host not in machines.split(','):
+ continue # skip this key
+
if I.startswith('ssh-'):
line = "%s %s"%(prefix, I)
else:
- line = "%s,%s"%(prefix, I)
+ continue # do not allow keys with other restrictions that might conflict
+ line = Sanitize(line) + "\n"
+ F.write(line)
+
+ for dn, attrs in hosts:
+ if 'sshRSAHostKey' not in attrs: continue
+ hostname = "host-" + attrs['hostname'][0]
+ prefix = GitoliteSSHRestrictions
+ prefix = prefix.replace('@@COMMAND@@', sshcommand)
+ prefix = prefix.replace('@@USER@@', hostname)
+ for I in attrs["sshRSAHostKey"]:
+ line = "%s %s"%(prefix, I)
line = Sanitize(line) + "\n"
F.write(line)
# Fetch all the users
userkeys = {}
- safe_rmtree(os.path.join(global_dir, 'userkeys'))
- safe_makedirs(os.path.join(global_dir, 'userkeys'))
-
for a in accounts:
- if not 'sshRSAAuthKey' in a: continue
+ if 'sshRSAAuthKey' not in a: continue
contents = []
for I in a['sshRSAAuthKey']:
os.umask(OldMask)
for a in accounts:
- if not 'webPassword' in a: continue
+ if 'webPassword' not in a: continue
if not a.pw_active(): continue
Pass = str(a['webPassword'])
Die(File, None, F)
raise
-def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target):
+# Generate the rtcPassword list
+def GenRtcPassword(accounts, File):
+ F = None
+ try:
+ OldMask = os.umask(0077)
+ F = open(File, "w", 0600)
+ os.umask(OldMask)
+
+ for a in accounts:
+ if a.is_guest_account(): continue
+ if 'rtcPassword' not in a: continue
+ if not a.pw_active(): continue
+
+ Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
+ Line = Sanitize(Line) + "\n"
+ F.write("%s" % (Line))
+
+ except:
+ Die(File, None, F)
+ raise
+
+# Generate the TOTP auth file
+def GenTOTPSeed(accounts, File):
+ F = None
+ try:
+ OldMask = os.umask(0077)
+ F = open(File, "w", 0600)
+ os.umask(OldMask)
+
+ F.write("# Option User Prefix Seed\n")
+ for a in accounts:
+ if a.is_guest_account(): continue
+ if 'totpSeed' not in a: continue
+ if not a.pw_active(): continue
+
+ Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
+ Line = Sanitize(Line) + "\n"
+ F.write("%s" % (Line))
+ except:
+ Die(File, None, F)
+ raise
+
+
+def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
OldMask = os.umask(0077)
- tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % CurrentHost), mode='w:gz')
+ tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
os.umask(OldMask)
for f in userlist:
if f not in ssh_userkeys:
pass
if grname is None:
- print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, CurrentHost, userlist[f])
+ print "User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f])
continue
lines = []
for line in ssh_userkeys[f]:
if line.startswith("allowed_hosts=") and ' ' in line:
machines, line = line.split('=', 1)[1].split(' ', 1)
- if CurrentHost not in machines.split(','):
+ if current_host not in machines.split(','):
continue # skip this key
lines.append(line)
if not lines:
tf.addfile(to, StringIO(contents))
tf.close()
- os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % CurrentHost), target)
+ os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)
# add a list of groups to existing groups,
# including all subgroups thereof, recursively.
# basically this proceduces the transitive hull of the groups in
# addgroups.
-def addGroups(existingGroups, newGroups, uid):
+def addGroups(existingGroups, newGroups, uid, current_host):
for group in newGroups:
# if it's a <group>@host, split it and verify it's on the current host.
s = group.split('@', 1)
- if len(s) == 2 and s[1] != CurrentHost:
+ if len(s) == 2 and s[1] != current_host:
continue
group = s[0]
existingGroups.append(group)
if SubGroupMap.has_key(group):
- addGroups(existingGroups, SubGroupMap[group], uid)
+ addGroups(existingGroups, SubGroupMap[group], uid, current_host)
# Generate the group list
-def GenGroup(accounts, File):
+def GenGroup(accounts, File, current_host):
grouprevmap = {}
F = None
try:
# Generate the GroupMap
GroupMap = {}
- for x in GroupIDMap.keys():
+ for x in GroupIDMap:
GroupMap[x] = []
GroupHasPrimaryMembers = {}
# Sort them into a list of groups having a set of users
for a in accounts:
GroupHasPrimaryMembers[ a['gidNumber'] ] = True
- if not 'supplementaryGid' in a: continue
+ if 'supplementaryGid' not in a: continue
supgroups=[]
- addGroups(supgroups, a['supplementaryGid'], a['uid'])
+ addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
for g in supgroups:
GroupMap[g].append(a['uid'])
# Output the group file.
J = 0
for x in GroupMap.keys():
- if GroupIDMap.has_key(x) == 0:
+ if x not in GroupIDMap:
continue
if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
def CheckForward(accounts):
for a in accounts:
- if not 'emailForward' in a: continue
+ if 'emailForward' not in a: continue
delete = False
os.umask(OldMask)
for a in accounts:
- if not 'emailForward' in a: continue
+ if 'emailForward' not in a: continue
Line = "%s: %s" % (a['uid'], a['emailForward'])
Line = Sanitize(Line) + "\n"
F.write(Line)
Done(File, F, None)
def GenCDB(accounts, File, key):
+ prefix = ["/usr/bin/eatmydata"] if os.path.exists('/usr/bin/eatmydata') else []
+ # nothing else does the fsync stuff, so why do it here?
+ Fdb = subprocess.Popen(prefix + ["cdbmake", File, "%s.tmp" % File],
+ preexec_fn=lambda: os.umask(0022),
+ stdin=subprocess.PIPE)
+ try:
+ # Write out the email address for each user
+ for a in accounts:
+ if key not in a: continue
+ value = a[key]
+ user = a['uid']
+ Fdb.stdin.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
+
+ Fdb.stdin.write("\n")
+ finally:
+ Fdb.stdin.close()
+ if Fdb.wait() != 0:
+ raise Exception("cdbmake gave an error")
+
+def GenDBM(accounts, File, key):
Fdb = None
+ OldMask = os.umask(0022)
+ fn = os.path.join(File).encode('ascii', 'ignore')
try:
- OldMask = os.umask(0022)
- Fdb = os.popen("cdbmake %s %s.tmp"%(File, File), "w")
+ posix.remove(fn)
+ except:
+ pass
+
+ try:
+ Fdb = dbm.open(fn, "c")
os.umask(OldMask)
# Write out the email address for each user
for a in accounts:
- if not key in a: continue
+ if key not in a: continue
value = a[key]
user = a['uid']
- Fdb.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))
+ Fdb[user] = value
- Fdb.write("\n")
- # Oops, something unspeakable happened.
- except:
Fdb.close()
+ except:
+ # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
+ os.remove(File + ".db")
raise
- if Fdb.close() != None:
- raise "cdbmake gave an error"
+ # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
+ os.rename (File + ".db", File)
# Generate the anon XEarth marker file
def GenMarkers(accounts, File):
# Write out the position for each user
for a in accounts:
if not a.is_active_user(): continue
- if not 'privateSub' in a: continue
+ if a.is_guest_account(): continue
+ if 'privateSub' not in a: continue
try:
Line = "%s"%(a['privateSub'])
Line = Sanitize(Line) + "\n"
F = open(File + ".tmp", "w")
for a in accounts:
- if not 'mailDisableMessage' in a: continue
+ if 'mailDisableMessage' not in a: continue
Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
Line = Sanitize(Line) + "\n"
F.write(Line)
F = open(File + ".tmp", "w")
for a in accounts:
- if not key in a: continue
+ if key not in a: continue
if not a[key] == 'TRUE': continue
Line = "%s"%(a['uid'])
Line = Sanitize(Line) + "\n"
else: validregex = re.compile('^[-\w.]+$')
for a in accounts:
- if not key in a: continue
+ if key not in a: continue
filtered = filter(lambda z: validregex.match(z), a[key])
if len(filtered) == 0: continue
# Write out the zone file entry for each user
for a in accounts:
- if not 'dnsZoneEntry' in a: continue
+ if 'dnsZoneEntry' not in a: continue
if not a.is_active_user() and not isRoleAccount(a): continue
+ if a.is_guest_account(): continue
try:
F.write("; %s\n"%(a.email_address()))
F.write(Line)
Host = Split[0] + DNSZone
- if BSMTPCheck.match(Line) != None:
+ if BSMTPCheck.match(Line) is not None:
F.write("; Has BSMTP\n")
# Write some identification information
raise
Done(File, F, None)
+def is_ipv6_addr(i):
+ try:
+ socket.inet_pton(socket.AF_INET6, i)
+ except socket.error:
+ return False
+ return True
+
def ExtractDNSInfo(x):
+ hostname = GetAttr(x, "hostname")
TTLprefix="\t"
if 'dnsTTL' in x[1]:
DNSInfo = []
if x[1].has_key("ipHostNumber"):
for I in x[1]["ipHostNumber"]:
- if IsV6Addr.match(I) != None:
- DNSInfo.append("%sIN\tAAAA\t%s" % (TTLprefix, I))
+ if is_ipv6_addr(I):
+ DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I))
else:
- DNSInfo.append("%sIN\tA\t%s" % (TTLprefix, I))
+ DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I))
Algorithm = None
+ ssh_hostnames = [ hostname ]
+ if x[1].has_key("sshfpHostname"):
+ ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ]
+
if 'sshRSAHostKey' in x[1]:
for I in x[1]["sshRSAHostKey"]:
Split = I.split()
- if Split[0] == 'ssh-rsa':
+ key_prefix = Split[0]
+ key = base64.decodestring(Split[1])
+
+ # RFC4255
+ # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
+ if key_prefix == 'ssh-rsa':
Algorithm = 1
- if Split[0] == 'ssh-dss':
+ if key_prefix == 'ssh-dss':
Algorithm = 2
- if Algorithm == None:
+ if key_prefix == 'ssh-ed25519':
+ Algorithm = 4
+ if Algorithm is None:
continue
- Fingerprint = hashlib.new('sha1', base64.decodestring(Split[1])).hexdigest()
- DNSInfo.append("%sIN\tSSHFP\t%u 1 %s" % (TTLprefix, Algorithm, Fingerprint))
+ # and more from the registry
+ sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ]
+
+ fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ]
+ for h in ssh_hostnames:
+ for digest_codepoint, fingerprint in fingerprints:
+ DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))
if 'architecture' in x[1]:
Arch = GetAttr(x, "architecture")
Mach = ""
if x[1].has_key("machine"):
Mach = " " + GetAttr(x, "machine")
- DNSInfo.append("%sIN\tHINFO\t\"%s%s\" \"%s\"" % (TTLprefix, Arch, Mach, "Debian GNU/Linux"))
+ DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))
if x[1].has_key("mXRecord"):
for I in x[1]["mXRecord"]:
- DNSInfo.append("%sIN\tMX\t%s" % (TTLprefix, I))
+ if I in MX_remap:
+ for e in MX_remap[I]:
+ DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))
+ else:
+ DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I))
return DNSInfo
if IsDebianHost.match(GetAttr(x, "hostname")) is None:
continue
- DNSInfo = ExtractDNSInfo(x)
- start = True
- for Line in DNSInfo:
- if start == True:
- Line = "%s.\t%s" % (GetAttr(x, "hostname"), Line)
- start = False
- else:
- Line = "\t\t\t%s" % (Line)
-
+ for Line in ExtractDNSInfo(x):
F.write(Line + "\n")
- # this would write sshfp lines for services on machines
- # but we can't yet, since some are cnames and we'll make
- # an invalid zonefile
- #
- # for i in x[1].get("purpose", []):
- # m = PurposeHostField.match(i)
- # if m:
- # m = m.group(1)
- # # we ignore [[*..]] entries
- # if m.startswith('*'):
- # continue
- # if m.startswith('-'):
- # m = m[1:]
- # if m:
- # if not m.endswith(HostDomain):
- # continue
- # if not m.endswith('.'):
- # m = m + "."
- # for Line in DNSInfo:
- # if isSSHFP.match(Line):
- # Line = "%s\t%s" % (m, Line)
- # F.write(Line + "\n")
-
# Oops, something unspeakable happened.
except:
Die(File, F, None)
# Write out the zone file entry for each user
for a in accounts:
- if not 'dnsZoneEntry' in a: continue
+ if 'dnsZoneEntry' not in a: continue
if not a.is_active_user(): continue
try:
Line = " ".join(Split) + "\n"
Host = Split[0] + DNSZone
- if BSMTPCheck.match(Line) != None:
+ if BSMTPCheck.match(Line) is not None:
F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
a['uid'], HomePrefix, a['uid'], Host))
if Host[1].has_key("ipHostNumber"):
for addr in Host[1]["ipHostNumber"]:
IPAdresses.append(addr)
- if IsV6Addr.match(addr) is None and mapped == "True":
+ if not is_ipv6_addr(addr) and mapped == "True":
IPAdresses.append("::ffff:"+addr)
return IPAdresses
# Generate the ssh known hosts file
-def GenSSHKnown(host_attrs, File, mode=None):
+def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
F = None
try:
OldMask = os.umask(0022)
hosts = HostToIP(x)
if 'sshdistAuthKeysHost' in x[1]:
hosts += x[1]['sshdistAuthKeysHost']
- Line = 'command="rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (Host, ",".join(hosts), I)
+ clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host)
+ clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand)
+ Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I)
else:
Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I)
Line = Sanitize(Line) + "\n"
if IsDebianHost.match(GetAttr(x, "hostname")) is None:
continue
- if not 'ipHostNumber' in x[1]:
+ if 'ipHostNumber' not in x[1]:
continue
addrs = x[1]["ipHostNumber"]
"keyFingerPrint", "privateSub", "mailDisableMessage",\
"mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
"mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
- "mailContentInspectionAction", "webPassword"])
+ "mailContentInspectionAction", "webPassword", "rtcPassword",\
+ "bATVToken", "totpSeed", "mailDefaultOptions"])
if passwd_attrs is None:
raise UDEmptyList, "No Users"
# Fetch all the hosts
HostAttrs = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
- "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture"])
+ "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
+ "sshfpHostname"])
- if HostAttrs == None:
+ if HostAttrs is None:
raise UDEmptyList, "No Hosts"
HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))
return l
+
+
+def setup_group_maps(l):
+ # Fetch all the groups
+ group_id_map = {}
+ subgroup_map = {}
+ attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
+ ["gid", "gidNumber", "subGroup"])
+
+ # Generate the subgroup_map and group_id_map
+ for x in attrs:
+ if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":
+ continue
+ if x[1].has_key("gidNumber") == 0:
+ continue
+ group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
+ if x[1].has_key("subGroup") != 0:
+ subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])
+
+ global SubGroupMap
+ global GroupIDMap
+ SubGroupMap = subgroup_map
+ GroupIDMap = group_id_map
+
def generate_all(global_dir, ldap_conn):
accounts = get_accounts(ldap_conn)
host_attrs = get_hosts(ldap_conn)
accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")
accounts = filter(lambda x: not IsRetired(x), accounts)
- #accounts_DDs = filter(lambda x: IsGidDebian(x), accounts)
CheckForward(accounts)
GenMailDisable(accounts, global_dir + "mail-disable")
GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
+ GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
+ GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
+ GenCDB(accounts, global_dir + "default-mail-options.cdb", 'mailDefaultOptions')
+ GenDBM(accounts, global_dir + "default-mail-options.db", 'mailDefaultOptions')
GenPrivate(accounts, global_dir + "debian-private")
- GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys')
+ GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
GenWebPassword(accounts, global_dir + "web-passwords")
+ GenRtcPassword(accounts, global_dir + "rtc-passwords")
+ GenTOTPSeed(accounts, global_dir + "users.oath")
GenKeyrings(global_dir)
# Compatibility.
GenForward(accounts, global_dir + "forward-alias")
GenAllUsers(accounts, global_dir + 'all-accounts.json')
- accounts = filter(lambda a: not a in accounts_disabled, accounts)
+ accounts = filter(lambda a: a not in accounts_disabled, accounts)
ssh_userkeys = GenSSHShadow(global_dir, accounts)
GenMarkers(accounts, global_dir + "markers")
GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
GenHosts(host_attrs, global_dir + "debianhosts")
- GenSSHGitolite(accounts, global_dir + "ssh-gitolite")
GenDNS(accounts, global_dir + "dns-zone")
GenZoneRecords(host_attrs, global_dir + "dns-sshfp")
+ setup_group_maps(ldap_conn)
+
for host in host_attrs:
- if not "hostname" in host[1]:
+ if "hostname" not in host[1]:
continue
- generate_host(host, global_dir, accounts, ssh_userkeys)
+ generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)
-def generate_host(host, global_dir, accounts, ssh_userkeys):
- global CurrentHost
-
- CurrentHost = host[1]['hostname'][0]
- OutDir = global_dir + CurrentHost + '/'
- try:
+def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
+ current_host = host[1]['hostname'][0]
+ OutDir = global_dir + current_host + '/'
+ if not os.path.isdir(OutDir):
os.mkdir(OutDir)
- except:
- pass
# Get the group list and convert any named groups to numerics
GroupList = {}
ExtraList[extra.upper()] = True
if GroupList != {}:
- accounts = filter(lambda x: IsInGroup(x, GroupList), accounts)
+ accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)
DoLink(global_dir, OutDir, "debianhosts")
DoLink(global_dir, OutDir, "ssh_known_hosts")
else:
userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
sys.stdout.flush()
- grouprevmap = GenGroup(accounts, OutDir + "group")
- GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList))
+ grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
+ GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)
# Now we know who we're allowing on the machine, export
# the relevant ssh keys
- GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'))
+ GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)
- if not 'NOPASSWD' in ExtraList:
+ if 'NOPASSWD' not in ExtraList:
GenShadow(accounts, OutDir + "shadow")
# Link in global things
- if not 'NOMARKERS' in ExtraList:
+ if 'NOMARKERS' not in ExtraList:
DoLink(global_dir, OutDir, "markers")
DoLink(global_dir, OutDir, "mail-forward.cdb")
+ DoLink(global_dir, OutDir, "mail-forward.db")
DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
+ DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
DoLink(global_dir, OutDir, "mail-disable")
DoLink(global_dir, OutDir, "mail-greylist")
DoLink(global_dir, OutDir, "mail-callout")
DoLink(global_dir, OutDir, "mail-rhsbl")
DoLink(global_dir, OutDir, "mail-whitelist")
DoLink(global_dir, OutDir, "all-accounts.json")
+ DoLink(global_dir, Outdir, "default-mail-options.cdb")
+ DoLink(global_dir, Outdir, "default-mail-options.db")
GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
+ GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
- GenCDB(accounts, OutDir + "default-mail-options.cdb", 'mailDefaultOptions')
+ GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')
# Compatibility.
DoLink(global_dir, OutDir, "forward-alias")
DoLink(global_dir, OutDir, "debian-private")
if 'GITOLITE' in ExtraList:
- DoLink(global_dir, OutDir, "ssh-gitolite")
+ GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)
+ if 'exportOptions' in host[1]:
+ for entry in host[1]['exportOptions']:
+ v = entry.split('=',1)
+ if v[0] != 'GITOLITE' or len(v) != 2: continue
+ options = v[1].split(',')
+ group = options.pop(0);
+ gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
+ if 'nohosts' not in options:
+ gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
+ else:
+ gitolite_hosts = []
+ command = None
+ for opt in options:
+ if opt.startswith('sshcmd='):
+ command = opt.split('=',1)[1]
+ GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)
if 'WEB-PASSWORDS' in ExtraList:
DoLink(global_dir, OutDir, "web-passwords")
+ if 'RTC-PASSWORDS' in ExtraList:
+ DoLink(global_dir, OutDir, "rtc-passwords")
+
+ if 'TOTP' in ExtraList:
+ DoLink(global_dir, OutDir, "users.oath")
+
if 'KEYRING' in ExtraList:
for k in Keyrings:
bn = os.path.basename(k)
return last
+def getLastKeyringChangeTime():
+ krmod = 0
+ for k in Keyrings:
+ mt = os.path.getmtime(k)
+ if mt > krmod:
+ krmod = mt
+
+ return int(krmod)
+
def getLastBuildTime(gdir):
- cache_last_mod = 0
+ cache_last_ldap_mod = 0
+ cache_last_unix_mod = 0
+ cache_last_run = 0
try:
fd = open(os.path.join(gdir, "last_update.trace"), "r")
cache_last_mod=fd.read().split()
try:
- cache_last_mod = cache_last_mod[0]
- except IndexError:
+ cache_last_ldap_mod = cache_last_mod[0]
+ cache_last_unix_mod = int(cache_last_mod[1])
+ cache_last_run = int(cache_last_mod[2])
+ except IndexError, ValueError:
pass
fd.close()
except IOError, e:
else:
raise e
- return cache_last_mod
-
-
-
+ return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)
+
+def mq_notify(options, message):
+ options.section = 'dsa-udgenerate'
+ options.config = '/etc/dsa/pubsub.conf'
+
+ config = Config(options)
+ conf = {
+ 'rabbit_userid': config.username,
+ 'rabbit_password': config.password,
+ 'rabbit_virtual_host': config.vhost,
+ 'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
+ 'use_ssl': False
+ }
+
+ msg = {
+ 'message': message,
+ 'timestamp': int(time.time())
+ }
+ conn = None
+ try:
+ conn = Connection(conf=conf)
+ conn.topic_send(config.topic,
+ json.dumps(msg),
+ exchange_name=config.exchange,
+ timeout=5)
+ finally:
+ if conn:
+ conn.close()
def ud_generate():
- global GroupIDMap
parser = optparse.OptionParser()
parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
help="Output directory.")
parser.add_option("-f", "--force", dest="force", action="store_true",
- help="Force generation, even if not update to LDAP has happened.")
+ help="Force generation, even if no update to LDAP has happened.")
(options, args) = parser.parse_args()
if len(args) > 0:
parser.print_help()
sys.exit(1)
-
- l = make_ldap_conn()
-
if options.generatedir is not None:
generate_dir = os.environ['UD_GENERATEDIR']
elif 'UD_GENERATEDIR' in os.environ:
generate_dir = os.environ['UD_GENERATEDIR']
+ else:
+ generate_dir = GenerateDir
- ldap_last_mod = getLastLDAPChangeTime(l)
- cache_last_mod = getLastBuildTime(generate_dir)
- need_update = ldap_last_mod > cache_last_mod
- if not options.force and not need_update:
- fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
- fd.write("%s\n%s\n" % (ldap_last_mod, int(time.time())))
- fd.close()
- sys.exit(0)
+ lockf = os.path.join(generate_dir, 'ud-generate.lock')
+ lock = get_lock( lockf )
+ if lock is None:
+ sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
+ sys.exit(1)
- # Fetch all the groups
- GroupIDMap = {}
- attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
- ["gid", "gidNumber", "subGroup"])
+ l = make_ldap_conn()
- # Generate the SubGroupMap and GroupIDMap
- for x in attrs:
- if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":
- continue
- if x[1].has_key("gidNumber") == 0:
- continue
- GroupIDMap[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
- if x[1].has_key("subGroup") != 0:
- SubGroupMap.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])
+ time_started = int(time.time())
+ ldap_last_mod = getLastLDAPChangeTime(l)
+ unix_last_mod = getLastKeyringChangeTime()
+ cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)
- lock = None
- try:
- lockf = os.path.join(generate_dir, 'ud-generate.lock')
- lock = get_lock( lockf )
- if lock is None:
- sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
- sys.exit(1)
+ need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)
- tracefd = open(os.path.join(generate_dir, "last_update.trace"), "w")
+ fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
+ if need_update or options.force:
+ msg = 'Update forced' if options.force else 'Update needed'
generate_all(generate_dir, l)
- tracefd.write("%s\n%s\n" % (ldap_last_mod, int(time.time())))
- tracefd.close()
+ if use_mq:
+ mq_notify(options, msg)
+ last_run = int(time.time())
+ fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
+ fd.close()
+ sys.exit(0)
- finally:
- if lock is not None:
- lock.release()
if __name__ == "__main__":
- ud_generate()
-
+ if 'UD_PROFILE' in os.environ:
+ import cProfile
+ import pstats
+ cProfile.run('ud_generate()', "udg_prof")
+ p = pstats.Stats('udg_prof')
+ ##p.sort_stats('time').print_stats()
+ p.sort_stats('cumulative').print_stats()
+ else:
+ ud_generate()
# vim:set et:
# vim:set ts=3: