From 1af943401cb3dea6a550030e3c6ad3043c463e03 Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Sun, 21 Apr 2013 20:48:36 +0200 Subject: [PATCH] Add new porterbox class --- modules/debian-org/misc/local.yaml | 4 + modules/porterbox/files/dd-schroot-cmd | 217 ++++++++++++++++++ modules/porterbox/files/schroot-dsa/config | 30 +++ .../99porterbox-extra-apt-options | 35 +++ .../schroot-setup.d/99porterbox-extra-sources | 57 +++++ modules/porterbox/files/setup-dchroot | 206 +++++++++++++++++ modules/porterbox/manifests/init.pp | 26 +++ .../porterbox/templates/default-mirror.erb | 8 + modules/roles/manifests/init.pp | 4 + modules/sudo/files/sudoers | 2 +- 10 files changed, 588 insertions(+), 1 deletion(-) create mode 100755 modules/porterbox/files/dd-schroot-cmd create mode 100644 modules/porterbox/files/schroot-dsa/config create mode 100755 modules/porterbox/files/schroot-setup.d/99porterbox-extra-apt-options create mode 100755 modules/porterbox/files/schroot-setup.d/99porterbox-extra-sources create mode 100755 modules/porterbox/files/setup-dchroot create mode 100644 modules/porterbox/manifests/init.pp create mode 100644 modules/porterbox/templates/default-mirror.erb diff --git a/modules/debian-org/misc/local.yaml b/modules/debian-org/misc/local.yaml index dcfbfe2a5..297c759ce 100644 --- a/modules/debian-org/misc/local.yaml +++ b/modules/debian-org/misc/local.yaml @@ -190,6 +190,10 @@ services: - wolkenstein.debian.org keyring: - kaufmann.debian.org + # new style porterbox (schroot, dd installable packages etc) + porterbox: + - barriere.debian.org + - fischer.debian.org host_settings: heavy_exim: - bellini.debian.org diff --git a/modules/porterbox/files/dd-schroot-cmd b/modules/porterbox/files/dd-schroot-cmd new file mode 100755 index 000000000..e82a71c88 --- /dev/null +++ b/modules/porterbox/files/dd-schroot-cmd @@ -0,0 +1,217 @@ +#!/usr/bin/python + +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## + + +# Copyright (c) 2013 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# script to allow otherwise unprivileged users to do certain +# apt commands in schroot environments. + +# bugs: +# - ownership of the schroot session is only checked at the beginning. +# This means that if the original user deleted it, and then somebody +# else comes along and creates a session of the same name, they might +# get some of our commands run in there. + +import ConfigParser +import optparse +import os +import pipes +import platform +import pty +import re +import stat +import subprocess +import sys +from errno import EIO + +SCHROOT_SUPER_UID = 0 +SCHROOT_SUPER = 'root' + +def die(s): + print >> sys.stderr, s + sys.exit(1) + +def get_session_owner(session): + if re.search('[^0-9a-zA-Z_-]', session): + die("Invalid session name.") + + path = os.path.join('/var/lib/schroot/session', session) + config = ConfigParser.RawConfigParser() + config.read(path) + owner = [] + try: + owner.append(config.get(session, 'users')) + owner.append(config.get(session, 'root-users')) + except ConfigParser.NoSectionError: + die("Did not find session definition in session file.") + except ConfigParser.NoOptionError: + die("Did not find user information in session file.") + return owner + + +def ensure_ok(session): + if 'SUDO_USER' not in os.environ: + die("Cannot find SUDO_USER in environment.") + if not os.environ['SUDO_USER'] in get_session_owner(session): + die("Session owner mismatch.") + +def os_supports_unshare(): + if platform.uname()[0] == 'GNU/kFreeBSD': + return False + return True + +class WrappedRunner(): + def __init__(self, session, args, unshare=True): + self.unshare = unshare + if not os_supports_unshare(): self.unshare = False + s,r = self.run('schroot', '-c', session, '--run-session', '--', 'env', 'DEBIAN_FRONTEND=noninteractive', *args) + if s != 0: + die("Command %s exited due to signal %d."%(' '.join(args), s)) + if r != 0: + die("Command %s exited with exit code %d."%(' '.join(args), r)) + + @staticmethod + def get_ret(status): + signal = status & 0xff + if signal == 0: retcode = status > 8 + else: retcode = 0 + return signal, retcode + + def run(self, *cmd): + if self.unshare: + cmdstr = ' '.join(pipes.quote(s) for s in cmd) + cmd = ['unshare', '--uts', '--ipc', '--net', '--'] + cmd += ['sh', '-c', 'ip addr add 127.0.0.1/8 dev lo && ip link set dev lo up && %s'%(cmdstr)] + pid, fd = pty.fork() + if pid == pty.CHILD: + fd = os.open("/dev/null", os.O_RDWR) + os.dup2(fd, 0) # stdin + os.execlp(cmd[0], *cmd) + try: + while 1: + b = os.read(fd, 1) + if b == "": break + sys.stdout.write(b) + except OSError, e: + if e[0] == EIO: pass + else: raise + os.close(fd) + p,v = os.waitpid(pid, 0) + s,r = WrappedRunner.get_ret(v) + return s,r + +class AptSchroot: + APT_DRY = ['apt-get', '--dry-run'] + APT_REAL = ['apt-get', '--assume-yes', '-o', 'Dpkg::Options::=--force-confnew'] + + def __init__(self, options, args): + self.session = options.chroot + self.assume_yes = options.assume_yes + if len(args) < 1: + die("No operation given for apt.") + op = args.pop(0) + self.args = args + + if op == "update": + self.ensure_no_extra_args() + self.apt_update() + elif op == "upgrade": + self.ensure_no_extra_args() + self.apt_upgrade() + elif op == "dist-upgrade": + self.ensure_no_extra_args() + self.apt_dist_upgrade() + elif op == "install": + self.apt_install(args) + elif op == "build-dep": + self.apt_build_dep(args) + else: + die("Invalid operation %s"%(op,)) + + def ensure_no_extra_args(self): + if len(self.args) > 0: + die("superfluous arguments: %s"%(' '.join(self.args),)) + + def apt_update(self): + self.secure_run(AptSchroot.APT_REAL +['update'], unshare=False) + + def apt_upgrade(self): + self.apt_simulate_and_ask(['upgrade']) + + def apt_dist_upgrade(self): + self.apt_simulate_and_ask(['dist-upgrade']) + + def apt_install(self, packages): + self.apt_simulate_and_ask(['install', '--'] + packages) + + def apt_build_dep(self, packages): + self.apt_simulate_and_ask(['build-dep', '--'] + packages) + + def apt_simulate_and_ask(self, cmd, split_download=True, run_clean=True): + if not self.assume_yes: + self.secure_run(AptSchroot.APT_DRY + cmd) + ans = raw_input("Do it for real [Y/n]: ") + if ans.lower() == 'n': sys.exit(0) + if split_download: + self.secure_run(AptSchroot.APT_REAL + ['--download-only'] + cmd, unshare=False) + self.secure_run(AptSchroot.APT_REAL + cmd) + if run_clean: + self.secure_run(AptSchroot.APT_REAL + ['clean']) + + def secure_run(self, args, unshare=True): + WrappedRunner(self.session, args, unshare) + + +parser = optparse.OptionParser() +parser.set_usage("""%prog [options] -c [-y] -- + Available commands: + apt-get update + apt-get upgrade + apt-get dist-upgrade + apt-get install ... + apt-get build-dep ...""") +parser.add_option("-c", "--chroot", metavar="chroot", dest="chroot", + help="Which chroot to act on") +parser.add_option("-y", "--assume-yes", dest="assume_yes", default=False, + action="store_true", help="Assume yes on confirm questions.") + +(options, args) = parser.parse_args() + +if len(args) < 1 or options.chroot is None: + parser.print_help() + sys.exit(1) + +if os.getuid() != SCHROOT_SUPER_UID: + os.execlp('sudo', 'sudo', '-u', SCHROOT_SUPER, '--', *sys.argv) + +ensure_ok(options.chroot) + +command = args.pop(0) +if command == "apt-get": + AptSchroot(options, args) +else: + die("Invalid command: %s."%(command,)) diff --git a/modules/porterbox/files/schroot-dsa/config b/modules/porterbox/files/schroot-dsa/config new file mode 100644 index 000000000..8ee7b635b --- /dev/null +++ b/modules/porterbox/files/schroot-dsa/config @@ -0,0 +1,30 @@ +# Settings for "dsa" profile. + +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## + +. "/etc/schroot/default/config" + +# added by weasel: +if [ "$CHROOT_SESSION_PURGE" = "true" ]; then + case $CHROOT_NAME in + *-*_*-dchroot-*) + suitevariant_arch_tuple=${CHROOT_NAME%%-dchroot-*} + suitevariant=${suitevariant_arch_tuple%_*} + SUITE_BASE=${suitevariant%-*} + SUITE_VARIANT=${suitevariant##*-} + ;; + *_*-dchroot-*) + SUITE_BASE=${CHROOT_NAME%%_*} + ;; + esac +fi + +CHROOT_FILE_UNPACK_DIR=/srv/chroot/schroot-unpack + +if [ -e /etc/schroot/dsa/default-mirror ]; then + MIRROR=$(cat /etc/schroot/dsa/default-mirror ) +fi +MIRROR=${MIRROR:-cdn.debian.net} diff --git a/modules/porterbox/files/schroot-setup.d/99porterbox-extra-apt-options b/modules/porterbox/files/schroot-setup.d/99porterbox-extra-apt-options new file mode 100755 index 000000000..26947b79d --- /dev/null +++ b/modules/porterbox/files/schroot-setup.d/99porterbox-extra-apt-options @@ -0,0 +1,35 @@ +#!/bin/bash + +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## + +# by weasel + +set -e + +. "$SETUP_DATA_DIR/common-data" +. "$SETUP_DATA_DIR/common-functions" + +if [ -f "$CHROOT_SCRIPT_CONFIG" ]; then + . "$CHROOT_SCRIPT_CONFIG" +elif [ "$STATUS" = "ok" ]; then + fatal "script-config file '$CHROOT_SCRIPT_CONFIG' does not exist" +fi + + +if [ "$1" = "setup-start" ] || [ "$1" = "setup-recover" ]; then + TGT="${CHROOT_PATH}/etc/apt/apt.conf.d/local-schroot" + rm -f "$TGT" + cat > "$TGT" << EOF +APT::Install-Recommends 0; +Acquire::http::Pipeline-Depth "0"; +Acquire::Languages "none"; +Acquire::PDiffs "false"; +EOF +#Acquire { +# http { Proxy "http://10.213.12.1:3128/"; }; +# ftp { Proxy "http://10.213.12.1:3128/"; }; +#} +fi diff --git a/modules/porterbox/files/schroot-setup.d/99porterbox-extra-sources b/modules/porterbox/files/schroot-setup.d/99porterbox-extra-sources new file mode 100755 index 000000000..60223a423 --- /dev/null +++ b/modules/porterbox/files/schroot-setup.d/99porterbox-extra-sources @@ -0,0 +1,57 @@ +#!/bin/bash + +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## + +# by weasel + +set -e + +. "$SETUP_DATA_DIR/common-data" +. "$SETUP_DATA_DIR/common-functions" + +if [ -f "$CHROOT_SCRIPT_CONFIG" ]; then + . "$CHROOT_SCRIPT_CONFIG" +elif [ "$STATUS" = "ok" ]; then + fatal "script-config file '$CHROOT_SCRIPT_CONFIG' does not exist" +fi + + +if [ "$1" = "setup-start" ] || [ "$1" = "setup-recover" ]; then + SRCL="${CHROOT_PATH}/etc/apt/sources.list.d/auto.list" + rm -f "$SRCL" + mirror=${MIRROR:-cdn.debian.net} + + case "${SUITE_BASE:-}" in + experimental) + echo "deb http://$mirror/debian/ sid main" >> "$SRCL" + echo "deb-src http://$mirror/debian/ sid main" >> "$SRCL" + ;; + sid) + ;; + *) + echo "deb http://security.debian.org/ ${SUITE_BASE}/updates main" >> "$SRCL" + echo "deb-src http://security.debian.org/ ${SUITE_BASE}/updates main" >> "$SRCL" + ;; + esac + echo "deb http://$mirror/debian/ ${SUITE_BASE} main" >> "$SRCL" + echo "deb-src http://$mirror/debian/ ${SUITE_BASE} main" >> "$SRCL" + + case "${SUITE_VARIANT:-}" in + backports) + case "${SUITE_BASE:-}" in + squeeze) + echo "deb http://backports.debian.org/debian-backports/ ${SUITE_BASE}-${SUITE_VARIANT} main" >> "$SRCL" + echo "deb-src http://backports.debian.org/debian-backports/ ${SUITE_BASE}-${SUITE_VARIANT} main" >> "$SRCL" + ;; + *) + echo "deb http://$mirror/debian/ ${SUITE_BASE}-${SUITE_VARIANT} main" >> "$SRCL" + echo "deb-src http://$mirror/debian/ ${SUITE_BASE}-${SUITE_VARIANT} main" >> "$SRCL" + ;; + esac + ;; + esac + echo "You probably want to run apt-get update in this session chroot." +fi diff --git a/modules/porterbox/files/setup-dchroot b/modules/porterbox/files/setup-dchroot new file mode 100755 index 000000000..0cba6fa12 --- /dev/null +++ b/modules/porterbox/files/setup-dchroot @@ -0,0 +1,206 @@ +#!/bin/bash + +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## + +# Copyright (c) 2013 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +THISARCH=$(dpkg --print-architecture) + +usage() +{ +cat << EOF +usage: $0 + +OPTIONS: + -a ARCH debootstrap arch [$arch] + -m MIRROR http mirror to use [$mirror] + -b basedir place where to build the chroot/tarball [$basedir] + -f overwrite stuff. + -h this help. +EOF +} + +die() { + echo >&2 "$*" + exit 1 +} + +do_cleanup() { + local cnt + cnt=$((${#cleanup[*]}-1)) + for i in $(seq ${cnt} -1 0); do + ${cleanup[$i]} || true + done +} + +genschrootconf() { + local suite="$1"; shift + local arch="$1"; shift + local target="$1"; shift + local extra="${1:-}"; shift || true + + if [ -n "$extra" ]; then + local suite="${suite}-${extra}" + fi + +cat << EOF +[${suite}_${arch}-dchroot] +description=Debian $suite chroot for $arch +type=file +file=$target +groups=Debian,guest +root-groups=adm +source-groups=adm +source-root-groups=adm +script-config=dsa/config +EOF + + if [ "$THISARCH" = "$arch" ]; then + echo "aliases=$suite" + else + case "$arch" in + i386) + echo "personality=linux32" + ;; + esac + fi + echo + + case "$suite" in + sid) + genschrootconf "experimental" "$arch" "$target" + ;; + experimental) + : + ;; + *) + if [ -z "$extra" ]; then + genschrootconf "$suite" "$arch" "$target" "backports" + fi + esac +} + + +set -e +set -u + +arch="$THISARCH" +if [ -e /etc/schroot/dsa/default-mirror ]; then + mirror=$(cat /etc/schroot/dsa/default-mirror ) +fi +mirror="${mirror:-cdn.debian.net}" +force="" +basedir="/srv/chroot" +declare -a cleanup +trap do_cleanup EXIT + +while getopts "a:b:fhm:" OPTION +do + case $OPTION in + a) + arch="$OPTARG" + ;; + b) + basedir="$OPTARG" + ;; + f) + force="1" + ;; + h) + help + exit 0 + ;; + m) + mirror="$OPTARG" + ;; + *) + usage >&2 + exit 1 + ;; + esac +done +shift $(($OPTIND - 1)) + +if [ "$#" != 1 ]; then + usage >&2 + exit 1 +fi +suite="$1"; shift +tuple="${suite}_${arch}" + +[ -d "$basedir" ] || die "Error: $basedir does not exist (or is not a directory)." + +target="$basedir/$tuple.tar.gz" +! [ -e "$target" ] || [ -n "$force" ] || die "Error: $target already exists." + +schrootconfig="/etc/schroot/chroot.d/${tuple}-dchroot" +! [ -e "$schrootconfig" ] || [ -n "$force" ] || die "Error: $schrootconfig already exists." + + +# +# let's go +# +genschrootconf "$suite" "$arch" "$target" | tee "$schrootconfig" + +rootdir=$(mktemp -d "$basedir/create-$suite-XXXXXX") +cleanup+=("rm -r $rootdir") +cleanup+=("umount $rootdir/sys") + +set -x +debootstrap \ + --keyring /usr/share/keyrings/debian-archive-keyring.gpg \ + --include="apt" \ + --variant=buildd \ + --arch="$arch" \ + "$suite" "$rootdir" "http://$mirror/debian" +echo "$tuple" > $rootdir/etc/debian_chroot + +chroot "$rootdir" apt-get update +chroot "$rootdir" apt-get install -y --no-install-recommends policyrcd-script-zg2 +cat > "$rootdir/usr/local/sbin/policy-rc.d" << 'EOF' +#!/bin/sh + +# policy-rc.d script for chroots. +# Copyright (c) 2007 Peter Palfrader + +while true; do + case "$1" in + -*) shift ;; + makedev) exit 0;; + *) + echo "Not running services in chroot." + exit 101 + ;; + esac +done +EOF +chmod +x "$rootdir/usr/local/sbin/policy-rc.d" +chroot "$rootdir" apt-get install -y --no-install-recommends zsh locales-all build-essential vim fakeroot devscripts gdb +rm -f "$rootdir/etc/apt/sources.list" "$rootdir/etc/apt/sources.list.d/*" +umount "$rootdir/sys" || true + +(cd "$rootdir" && tar caf "$target" .) diff --git a/modules/porterbox/manifests/init.pp b/modules/porterbox/manifests/init.pp new file mode 100644 index 000000000..68c44b329 --- /dev/null +++ b/modules/porterbox/manifests/init.pp @@ -0,0 +1,26 @@ +class porterbox { + include schroot + + file { '/etc/schroot/dsa': + ensure => directory, + } + file { '/etc/schroot/dsa/config': + source => 'puppet:///modules/porterbox/schroot-dsa/config', + } + file { '/etc/schroot/setup.d/99porterbox-extra-apt-options': + source => 'puppet:///modules/porterbox/schroot-setup.d/99porterbox-extra-apt-options', + } + file { '/etc/schroot/setup.d/99porterbox-extra-sources': + source => 'puppet:///modules/porterbox/schroot-setup.d/99porterbox-extra-sources', + } + file { '/usr/local/bin/dd-schroot-cmd': + source => 'puppet:///modules/porterbox/dd-schroot-cmd', + } + file { '/usr/local/sbin/setup-dchroot': + source => 'puppet:///modules/porterbox/setup-dchroot', + } + + file { '/etc/schroot/dsa/default-mirror': + content => template('porterbox/default-mirror.erb'), + } +} diff --git a/modules/porterbox/templates/default-mirror.erb b/modules/porterbox/templates/default-mirror.erb new file mode 100644 index 000000000..df1dc9fb0 --- /dev/null +++ b/modules/porterbox/templates/default-mirror.erb @@ -0,0 +1,8 @@ +<%= +ninfo = scope.lookupvar('site::nodeinfo') +if ninfo['hoster'].has_key?('mirror-debian'): + ninfo['hoster']['mirror-debian'] +else + 'cdn.debian.net' +end +%> diff --git a/modules/roles/manifests/init.pp b/modules/roles/manifests/init.pp index 7ff4457af..a4a6cf08b 100644 --- a/modules/roles/manifests/init.pp +++ b/modules/roles/manifests/init.pp @@ -16,6 +16,10 @@ class roles { include buildd } + if getfromhash($site::nodeinfo, 'porterbox') { + include porterbox + } + if getfromhash($site::nodeinfo, 'bugs_mirror') { include roles::bugs_mirror } diff --git a/modules/sudo/files/sudoers b/modules/sudo/files/sudoers index 1d236584b..a44617d99 100644 --- a/modules/sudo/files/sudoers +++ b/modules/sudo/files/sudoers @@ -185,4 +185,4 @@ nagiosadm tchaikovsky=(root) NOPASSWD: /usr/sbin/service icinga reload %porter-ppc partch=(root) /usr/sbin/upgrade-porter-chroots, /usr/bin/apt-in-chroot %porter-s390 zelenka=(root) /usr/sbin/upgrade-porter-chroots, /usr/bin/apt-in-chroot %porter-sparc smetana,sperger=(root) /usr/sbin/upgrade-porter-chroots, /usr/bin/apt-in-chroot -%Debian,%guest barriere,fischer=(root) NOPASSWD: /usr/local/bin/apt-schroot +%Debian,%guest barriere,fischer=(root) NOPASSWD: /usr/local/bin/dd-schroot-cmd -- 2.20.1