From ff762dfdfbfdb5f5eead0b998c01a6e8feeeac67 Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Sun, 26 Feb 2017 22:26:44 +0100 Subject: [PATCH] Put the scripts we need for pg backups into puppet --- hieradata/common.yaml | 8 + .../files/postgresql_server/pg-backup-file | 77 +++++++++ .../pg-receive-file-from-backup | 161 ++++++++++++++++++ modules/roles/manifests/init.pp | 4 + modules/roles/manifests/postgresql_server.pp | 13 ++ .../postgresql_server/pg-backup-file.conf.erb | 5 + 6 files changed, 268 insertions(+) create mode 100755 modules/roles/files/postgresql_server/pg-backup-file create mode 100755 modules/roles/files/postgresql_server/pg-receive-file-from-backup create mode 100644 modules/roles/manifests/postgresql_server.pp create mode 100644 modules/roles/templates/postgresql_server/pg-backup-file.conf.erb diff --git a/hieradata/common.yaml b/hieradata/common.yaml index dad21a56d..85af2d066 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -304,3 +304,11 @@ roles: - quantz.debian.org - tchaikovsky.debian.org - wuiet.debian.org + postgresql_server: + - bmdb1.debian.org + - danzi.debian.org + - fasolo.debian.org + - melartin.debian.org + - seger.debian.org + - sibelius.debian.org + - vittoria.debian.org diff --git a/modules/roles/files/postgresql_server/pg-backup-file b/modules/roles/files/postgresql_server/pg-backup-file new file mode 100755 index 000000000..4b6f7d3d8 --- /dev/null +++ b/modules/roles/files/postgresql_server/pg-backup-file @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright (c) 2010, 2011, 2013, 2014 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# Archive a postgresql file (WAL or BASE) to the archive server $backuphost +# can be set in /etc/dsa/pg-backup-file.conf + +set -u + +backuphost=debbackup@storace +myhost="`hostname`" +self="`basename "$0"`[$$]" + +if [ "$#" != 3 ] ; then + echo >&2 "Usage: $self " + exit 1 +fi + +ssh_options="" +! [ -e /etc/dsa/pg-backup-file.conf ] || . /etc/dsa/pg-backup-file.conf + +cluster="$1" +what="$2" +file="$3" + +info() { + logger -p daemon.info -t "$self" "$1" +} + +croak() { + logger -s -p daemon.warn -t "$self" "$1" + exit 1 +} + +if ! [ -e "$file" ] ; then + croak "file $file does not exist" +fi + +size="`stat -c '%s' "$file"`" +checksum="" +bn="`basename "$file"`" +targetname="$cluster.$what.$bn" +logtuple="($myhost,$targetname,$size,$checksum)" + +for target in $backuphost; do + if [ "${target#/}" != "$target" ]; then + info "Archiving to $target: ($targetname,$size)" + cp "$file" "$target/$myhost-$targetname" + else + [ -n "$checksum" ] || checksum="`sha512sum "$file" | awk '{print $1}'`" + info "Archiving to $target: ($myhost,$targetname,$size,$checksum)" + + ssh -C "$target" $ssh_options $myhost store-file pg "$targetname" "$size" "$checksum" < "$file" + if [ "$?" != 0 ]; then + croak "remote store for $logtuple failed." + fi + fi +done diff --git a/modules/roles/files/postgresql_server/pg-receive-file-from-backup b/modules/roles/files/postgresql_server/pg-receive-file-from-backup new file mode 100755 index 000000000..1b1c1b354 --- /dev/null +++ b/modules/roles/files/postgresql_server/pg-receive-file-from-backup @@ -0,0 +1,161 @@ +#!/usr/bin/python + +# Copyright (c) 2010 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# copy a file from the backup server, where the server runs debbackup-ssh-wrap + +import sys +import os +import optparse +import re +import subprocess +import syslog +import tempfile +import stat +import hashlib + +backuphost="debbackup@storace.debian.org" +block_size = 4096 + +syslog.openlog(sys.argv[0], syslog.LOG_PID, syslog.LOG_DAEMON) + +def info(m): + syslog.syslog(syslog.LOG_INFO, m) +def croak(m): + syslog.syslog(syslog.LOG_WARNING, m) + print >> sys.stderr, m + sys.exit(1) + +sys.stdin.close() + +parser = optparse.OptionParser() +parser.set_usage("%prog [] ") +(options, args) = parser.parse_args() + +if len(args) != 3: + parser.print_help() + sys.exit(1) + +from_host = args.pop(0) +filename = args.pop(0) +target = args.pop(0) + +hostname = os.uname()[1] + +# open pipe to backup server +p = subprocess.Popen(["ssh", "-C", backuphost, hostname, "retrieve-file", "pg", from_host, filename], stdin=subprocess.PIPE, stdout=subprocess.PIPE) +p.stdin.close() +f = p.stdout + +# read Format line +line = f.readline().rstrip('\n') +if line != "Format: 1": + croak("Uknown format in line 1 (%s) when getting %s:%s from backup host"%(line, from_host, filename)) + +# read Status line +line = f.readline().rstrip('\n') +s = line.split(':', 1) +if len(s) != 2: + croak("Protocol violation when getting %s:%s from backup host (line 2)"%(from_host, filename)) +key = s[0] +if key != "Status": + croak("Protocol violation when getting %s:%s from backup host"%(from_host, filename)) +first = s[1].split()[0] +try: + code = int(first) +except ValueError: + croak("Invalid code '%s'"%(first)) + +if code == 404: + info("Wanted to get %s:%s from backup host, but file does not exist."%(from_host, filename)) + sys.exit(1) +elif code != 200: + info("Unknown code %d when trying to get %s:%s from backup host"%(code, from_host, filename)) + +# get rest of the headers +metadata = {} +while True: + line = f.readline().rstrip('\n') + if line == "": break + line = line.strip() + s = line.split(':', 1) + if len(s) != 2: + croak("Protocol violation when getting %s:%s from backup host"%(from_host, filename)) + key = s[0] + value = s[1].lstrip() + metadata[key] = value + +# verify we like them +for k in ['Size', 'SHA-512']: + if not k in metadata: + croak("Key %s not found in metadata when getting %s:%s from backup host"%(k, from_host, filename)) +try: + size = int(metadata['Size']) +except ValueError: + croak("Invalid size '%s'"%(size)) +checksum = metadata['SHA-512'] + +info("Getting %s:%s from backup host (size %d)."%(from_host, filename, size)) + +target=os.path.abspath(target) +targetdir=os.path.dirname(target) +tmp = tempfile.NamedTemporaryFile(dir=targetdir, prefix=".tmp.pg-receive-file-from-backup-%s:%s."%(from_host, filename)) + +# read file +running_size = 0 +digest = hashlib.sha512() +while True: + buf = f.read(block_size) + if not buf: break + digest.update(buf) + tmp.write(buf) + + running_size += len(buf) + if running_size > size: + croak("Size mismatch") +f.close() +p.wait() +tmp.flush() + +file_size = os.stat(tmp.name)[stat.ST_SIZE] + +# check size and checksum +if file_size != size: + croak("Size mismatch") +if file_size != running_size: + croak("Size mismatch. WTF.") +if checksum != digest.hexdigest(): + croak("Checksum mismatch. WTF.") + +# and try to link +try: + os.link(tmp.name, target) +except Exception, e: + croak("Failed at linking to target: %s"%(e)) + +tmp.close() +info("Successfully stored %s."%(target)) + + +# vim:set et: +# vim:set ts=4: +# vim:set shiftwidth=4: diff --git a/modules/roles/manifests/init.pp b/modules/roles/manifests/init.pp index d8118e426..88397c055 100644 --- a/modules/roles/manifests/init.pp +++ b/modules/roles/manifests/init.pp @@ -326,4 +326,8 @@ class roles { if has_role('cdimage-search') { include roles::cdimage_search } + + if has_role('postgresql_server') { + include roles::postgresql_server + } } diff --git a/modules/roles/manifests/postgresql_server.pp b/modules/roles/manifests/postgresql_server.pp new file mode 100644 index 000000000..e90c27ee0 --- /dev/null +++ b/modules/roles/manifests/postgresql_server.pp @@ -0,0 +1,13 @@ +class roles::postgresql_server { + file { "/usr/local/bin/pg-backup-file": + mode => 555, + source => "puppet:///modules/roles/postgresql_server/pg-backup-file", + } + file { "/usr/local/bin/pg-receive-file-from-backup": + mode => 555, + source => "puppet:///modules/roles/postgresql_server/pg-receive-file-from-backup", + } + file { "/etc/dsa/pg-backup-file.conf": + content => template('roles/postgresql_server/pg-backup-file.conf.erb'), + } +} diff --git a/modules/roles/templates/postgresql_server/pg-backup-file.conf.erb b/modules/roles/templates/postgresql_server/pg-backup-file.conf.erb new file mode 100644 index 000000000..1293d385c --- /dev/null +++ b/modules/roles/templates/postgresql_server/pg-backup-file.conf.erb @@ -0,0 +1,5 @@ +<%- if hostname == "sibelius" then -%> +# use ipv4 +ssh_options="-oAddressFamily=inet" +<%- end %> +backuphost="debbackup@backuphost debbackup@storace" -- 2.20.1