From e272b442942830a6eed50748eb63aa3b6318ca2b Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Tue, 24 Sep 2019 21:12:05 +0200 Subject: [PATCH] Split the director config coming from each node in two parts: one that comes from the client directly and one that goes via the storage --- data/common.yaml | 1 + modules/bacula/manifests/client.pp | 15 ++-- modules/bacula/manifests/client/storage.pp | 9 -- modules/bacula/manifests/director.pp | 5 +- modules/bacula/manifests/director/client.pp | 26 ++---- .../manifests/director/client_from_storage.pp | 52 +++++++++++ modules/bacula/manifests/init.pp | 5 -- modules/bacula/manifests/storage.pp | 8 +- modules/bacula/manifests/storage/client.pp | 14 ++- modules/bacula/templates/bacula-fd.conf.erb | 2 +- modules/bacula/templates/bacula-sd.conf.erb | 6 +- .../director/dir-per-client-from-storage.erb | 82 +++++++++++++++++ .../templates/director/dir-per-client.erb | 88 ++----------------- 13 files changed, 184 insertions(+), 129 deletions(-) delete mode 100644 modules/bacula/manifests/client/storage.pp create mode 100644 modules/bacula/manifests/director/client_from_storage.pp create mode 100644 modules/bacula/templates/director/dir-per-client-from-storage.erb diff --git a/data/common.yaml b/data/common.yaml index 27d8a53e5..51e1f130a 100644 --- a/data/common.yaml +++ b/data/common.yaml @@ -35,6 +35,7 @@ roles::dns_primary::allow_access: bacula::director::db_address: 'postgresql-manda-01.debian.org' bacula::director::db_port: 5432 bacula::director::db_sslca: '/etc/ssl/debian/certs/ca.crt' +bacula::director::pool_name: 'debian' bacula::client::director_server: dinis.debian.org bacula::client::storage_server: storace.debian.org diff --git a/modules/bacula/manifests/client.pp b/modules/bacula/manifests/client.pp index 8bf5daae3..2b7c35f08 100644 --- a/modules/bacula/manifests/client.pp +++ b/modules/bacula/manifests/client.pp @@ -19,13 +19,14 @@ class bacula::client( $reverse_ensure = $ensure ? { 'present' => 'absent', 'absent' => 'present' } if $ensure == 'present' { - @@bacula::storage::client { $::fqdn: - tag => "bacula::to-storage::${storage_server}" - } - @@bacula::director::client { $::fqdn: port_fd => $port_fd, - tag => "bacula::to-director::${director_server}" + tag => "bacula::to-director::${director_server}", + } + + @@bacula::storage::client { $::fqdn: + tag => "bacula::to-storage::${storage_server}", + director_server => $director_server, } @@concat::fragment { "bacula-dsa-client-list::${::fqdn}": @@ -37,13 +38,13 @@ class bacula::client( } # allow access from director - Ferm::Rule::Simple <<| tag == "bacula::director-to-fd::${bacula::bacula_director_address}" |>> { + Ferm::Rule::Simple <<| tag == "bacula::director-to-fd::${director_server}" |>> { port => $port_fd, } # get access to the storage @@ferm::rule::simple { "bacula::fd-to-storage::${::fqdn}": - tag => "bacula::fd-to-storage::${bacula::bacula_storage_address}", + tag => "bacula::fd-to-storage::${storage_server}", description => 'Allow bacula-fd access to the bacula-storage', chain => 'bacula-sd', saddr => $bacula::public_addresses, diff --git a/modules/bacula/manifests/client/storage.pp b/modules/bacula/manifests/client/storage.pp deleted file mode 100644 index 716483c8e..000000000 --- a/modules/bacula/manifests/client/storage.pp +++ /dev/null @@ -1,9 +0,0 @@ -# Bacula configuration for a client, pushed from the storage. -# -# This is stored config by a storage and then collected on the client. -# -define bacula::client::storage( -) { - include bacula::client - -} diff --git a/modules/bacula/manifests/director.pp b/modules/bacula/manifests/director.pp index 956374915..8ea5859ba 100644 --- a/modules/bacula/manifests/director.pp +++ b/modules/bacula/manifests/director.pp @@ -1,5 +1,6 @@ # our bacula director # +# @param pool_name A string to be used in pool names # @param db_address hostname of the postgres server for the catalog DB # @param db_port port of the postgres server for the catalog DB # @param db_name DB name for the catalog DB @@ -10,6 +11,7 @@ class bacula::director( Integer $db_port, String $db_name = 'bacula', String $db_user = 'bacula', + String $pool_name = 'bacula', Optional[String] $db_sslca = undef, ) inherits bacula { @@ -67,7 +69,8 @@ class bacula::director( notify => Exec['bacula-director reload'] } - Bacula::Director::Client<<| tag == "bacula::to-director::${::fqdn}" |>> + Bacula::Director::Client <<| tag == "bacula::to-director::${::fqdn}" |>> + Bacula::Director::Client_from_stroage<<| tag == "bacula::to-director::${::fqdn}" |>> package { 'bacula-console': ensure => installed; diff --git a/modules/bacula/manifests/director/client.pp b/modules/bacula/manifests/director/client.pp index 76b0947b9..402a99708 100644 --- a/modules/bacula/manifests/director/client.pp +++ b/modules/bacula/manifests/director/client.pp @@ -8,21 +8,18 @@ define bacula::director::client ( Integer $port_fd = 9102, String $client = $name, ) { - include bacula + include bacula::director - $bacula_pool_name = $bacula::bacula_pool_name - $bacula_filestor_name = $bacula::bacula_filestor_name - $bacula_filestor_device = $bacula::bacula_filestor_device - $bacula_storage_address = $bacula::bacula_storage_address - $bacula_storage_port = $bacula::bacula_storage_port - $bacula_storage_secret = $bacula::bacula_storage_secret - - $bacula_ca_path = $bacula::bacula_ca_path + $bacula_ca_path = $bacula::bacula_ca_path $bacula_ssl_client_cert = $bacula::bacula_ssl_client_cert $bacula_ssl_client_key = $bacula::bacula_ssl_client_key - $bacula_client_name = "${client}-fd" - $bacula_client_secret = hkdf('/etc/puppet/secret', "bacula-fd-${client}") + $client_name = "${client}-fd" + $client_secret = hkdf('/etc/puppet/secret', "bacula-fd-${client}") + + # we define this in both bacula::director::client_from_storage and + # bacula::director::client and it needs to match. + $pool_name = "${bacula::director::pool_name}-${client}" file { "/etc/bacula/conf.d/${client}.conf": content => template('bacula/director/dir-per-client.erb'), @@ -30,12 +27,5 @@ define bacula::director::client ( group => bacula, notify => Exec['bacula-director reload'] } - - file { "/etc/bacula/storages-list.d/${client}.storage": - content => "${bacula::bacula_filestor_name}-${client}\n", - mode => '0440', - group => bacula, - notify => Exec['bacula-director reload'] - } } diff --git a/modules/bacula/manifests/director/client_from_storage.pp b/modules/bacula/manifests/director/client_from_storage.pp new file mode 100644 index 000000000..5be90c3cf --- /dev/null +++ b/modules/bacula/manifests/director/client_from_storage.pp @@ -0,0 +1,52 @@ +# Bacula client config on the director +# +# This is stored config by a client, collected on the director +# +# @param client The name of the client (relevant for device names, media type names, etc.) +# +# @param storage_address Address of the storage daemon +# @param port_sd Port of the storage daemon +# @param storage_secret Shared secret between storage and director +# @param storage_device_name Device name on the storage daemon for this node's backups +# @param storage_media_type_name Media type name on the storage daemon for this node's backupse +define bacula::director::client_from_storage ( + String $storage_address, + Integer $port_sd, + String $storage_secret, + String $storage_device_name, + String $storage_media_type_name, + + String $client = $name, +) { + include bacula::director + + $bacula_ca_path = $bacula::bacula_ca_path + $bacula_ssl_client_cert = $bacula::bacula_ssl_client_cert + $bacula_ssl_client_key = $bacula::bacula_ssl_client_key + + # For historical reasons, we use the same string for + # director-internal storage name as we do for mediate + # type names. If we ever blow away the catalog and start + # again, we should probably pick a different string here + # (or there). + $storage_name = $storage_media_type_name + + # we define this in both bacula::director::client_from_storage and + # bacula::director::client and it needs to match. + $pool_name = "${bacula::director::pool_name}-${client}" + + file { "/etc/bacula/conf.d/${client}_storage.conf": + content => template('bacula/director/dir-per-client-from-storage.erb'), + mode => '0440', + group => bacula, + notify => Exec['bacula-director reload'] + } + + file { "/etc/bacula/storages-list.d/${client}.storage": + content => "${storage_name}\n", + mode => '0440', + group => bacula, + notify => Exec['bacula-director reload'] + } +} + diff --git a/modules/bacula/manifests/init.pp b/modules/bacula/manifests/init.pp index 0caa7dab8..1074c75c6 100644 --- a/modules/bacula/manifests/init.pp +++ b/modules/bacula/manifests/init.pp @@ -7,18 +7,13 @@ class bacula ( String $bacula_storage_name = 'debian-sd', String $bacula_client_name = "${::fqdn}-fd", String $bacula_monitor_name = 'debian-mon', - String $bacula_filestor_name = 'File', - String $bacula_filestor_device = 'FileStorage', - String $bacula_pool_name = 'debian', String $bacula_director_address = 'dinis.debian.org', Integer $bacula_director_port = 9101, String $bacula_storage_address = 'storace.debian.org', - Integer $bacula_storage_port = 9103, String $bacula_director_secret = hkdf('/etc/puppet/secret', "bacula-dir-${::hostname}"), String $bacula_db_secret = hkdf('/etc/puppet/secret', "bacula-db-${::hostname}"), - String $bacula_storage_secret = hkdf('/etc/puppet/secret', "bacula-sd-${bacula_storage_name}"), String $bacula_client_secret = hkdf('/etc/puppet/secret', "bacula-fd-${::fqdn}"), String $bacula_monitor_secret = hkdf('/etc/puppet/secret', "bacula-monitor-${bacula_director_name}"), diff --git a/modules/bacula/manifests/storage.pp b/modules/bacula/manifests/storage.pp index c2b6886d9..69e8ddc44 100644 --- a/modules/bacula/manifests/storage.pp +++ b/modules/bacula/manifests/storage.pp @@ -3,11 +3,15 @@ # @param backup_path the directory where backups should be stored # @param filestor_device Storage device name prefix # @param filestor_name Storage device media type name prefix +# @param port_sd Port for the sd to listen on class bacula::storage ( String $backup_path = '/srv/bacula', String $filestor_device = 'FileStorage', String $filestor_name = 'File', + Integer $port_sd = 9103, ) inherits bacula { + $storage_secret = hkdf('/etc/puppet/secret', "bacula-sd-${::fqdn}") + package { 'bacula-sd': ensure => installed } @@ -27,7 +31,7 @@ class bacula::storage ( exec { 'bacula-sd restart-when-idle': path => '/usr/bin:/usr/sbin:/bin:/sbin', - command => "sh -c 'setsid /usr/local/sbin/bacula-idle-restart ${bacula::bacula_storage_port} bacula-sd &'", + command => "sh -c 'setsid /usr/local/sbin/bacula-idle-restart ${port_sd} bacula-sd &'", refreshonly => true, subscribe => File[$bacula::bacula_ssl_server_cert], require => File['/usr/local/sbin/bacula-idle-restart'], @@ -55,7 +59,7 @@ class bacula::storage ( # allow access from director and fds ferm::rule::simple { 'dsa-bacula-sd': description => 'Access to the bacula-storage', - port => $bacula::bacula_storage_port, + port => $port_sd, target => 'bacula-sd', } Ferm::Rule::Simple <<| tag == "bacula::director-to-storage::${bacula::bacula_director_address}" |>>; diff --git a/modules/bacula/manifests/storage/client.pp b/modules/bacula/manifests/storage/client.pp index 7aa04287a..71d8731da 100644 --- a/modules/bacula/manifests/storage/client.pp +++ b/modules/bacula/manifests/storage/client.pp @@ -2,8 +2,10 @@ # # This is stored config by a client and then collected on the storage # -# @param client The name of the client (relevant for device names, media type names, etc.) +# @param director_server The director for this client +# @param client The name of the client (relevant for device names, media type names, etc.) define bacula::storage::client( + String $director_server, String $client = $name ) { include bacula::storage @@ -26,4 +28,14 @@ define bacula::storage::client( group => bacula, ; } + + # enable the director to make (client, storage) specific configuration + @@bacula::director::client_from_storage { $::fqdn: + tag => "bacula::to-director::${director_server}", + storage_address => $::fqdn, + port_sd => $bacula::storage::port_sd, + storage_secret => $bacula::storage::storage_secret, + storage_device_name => $device_name, + storage_media_type_name => $media_type_name, + } } diff --git a/modules/bacula/templates/bacula-fd.conf.erb b/modules/bacula/templates/bacula-fd.conf.erb index 547a06d9d..16a89e088 100644 --- a/modules/bacula/templates/bacula-fd.conf.erb +++ b/modules/bacula/templates/bacula-fd.conf.erb @@ -12,7 +12,7 @@ Director { TLS Enable = yes TLS Require = yes TLS Verify Peer = yes - TLS Allowed CN = "clientcerts/<%= @bacula_director_address %>" + TLS Allowed CN = "clientcerts/<%= @director_server %>" TLS CA Certificate File = "<%= @bacula_ca_path %>" # This is a server certificate, used for incoming director connections. TLS Certificate = "<%= @bacula_ssl_server_cert %>" diff --git a/modules/bacula/templates/bacula-sd.conf.erb b/modules/bacula/templates/bacula-sd.conf.erb index a46d9739b..f131f5a8e 100644 --- a/modules/bacula/templates/bacula-sd.conf.erb +++ b/modules/bacula/templates/bacula-sd.conf.erb @@ -15,13 +15,13 @@ Storage { # use the hostname rather than the IP address from LDAP, # as /etc/hosts might have a better answer in case of natted hosts. addr = <%= @bacula_storage_address %> - port = <%= @bacula_storage_port %> + port = <%= @port_sd %> } <%- end -%> <%- if scope.lookupvar('deprecated::nodeinfo')['misc']['has_v6_ldap'] -%> ipv6 = { addr = <%= @bacula_storage_address %> - port = <%= @bacula_storage_port %> + port = <%= @port_sd %> } <%- end -%> } @@ -44,7 +44,7 @@ Storage { # Director { Name = <%= @bacula_director_name %> - Password = "<%= @bacula_storage_secret %>" + Password = "<%= @storage_secret %>" TLS Enable = yes TLS Require = yes diff --git a/modules/bacula/templates/director/dir-per-client-from-storage.erb b/modules/bacula/templates/director/dir-per-client-from-storage.erb new file mode 100644 index 000000000..8a634430e --- /dev/null +++ b/modules/bacula/templates/director/dir-per-client-from-storage.erb @@ -0,0 +1,82 @@ +## +## THIS FILE IS UNDER PUPPET CONTROL. DON'T EDIT IT HERE. +## USE: git clone git+ssh://$USER@puppet.debian.org/srv/puppet.debian.org/git/dsa-puppet.git +## +# For Bacula release 5.0.1 (24 February 2010) -- debian 5.0.4 + +######################################################################## +# Storage config # +######################################################################## + +Storage { + Name = "<%= @storage_name %>" + Address = <%= @storage_address %> + SDPort = <%= @port_sd %> + Password = "<%= @storage_secret %>" + Device = "<%= @storage_device_name %>" + Media Type = "<%= @storage_media_type_name %>" + Maximum Concurrent Jobs = 10 + + TLS Enable = yes + TLS Require = yes + TLS CA Certificate File = "<%= @bacula_ca_path %>" + # This is a client certificate, used by the director to connect to the storage daemon + TLS Certificate = "<%= @bacula_ssl_client_cert %>" + TLS Key = "<%= @bacula_ssl_client_key %>" +} + +######################################################################## +# Pool definition # +######################################################################## +Pool { + Name = "poolfull-<%=@pool_name %>" + Pool Type = Backup + Storage = "<%= @storage_name %>" + AutoPrune = yes + Volume Retention = 100 days + Label Format = "<%= @pool_name %>-full.${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}_${Hour:p/2/0/r}:${Minute:p/2/0/r}" + Volume Use Duration = 23h + Maximum Volume Jobs = 1 + Maximum Volume Bytes = 500G + Action On Purge = Truncate + Recycle = yes + RecyclePool = "poolgraveyard-<%= @pool_name %>" +} + +Pool { + Name = "pooldiff-<%= @pool_name %>" + Pool Type = Backup + Storage = "<%= @storage_name %>" + AutoPrune = yes + Volume Retention = 50 days + Label Format = "<%= @pool_name %>-diff.${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}_${Hour:p/2/0/r}:${Minute:p/2/0/r}" + Volume Use Duration = 23h + Maximum Volume Jobs = 1 + Maximum Volume Bytes = 500G + Action On Purge = Truncate + Recycle = yes + RecyclePool = "poolgraveyard-<%= @pool_name %>" +} + +Pool { + Name = "poolinc-<%= @pool_name %>" + Pool Type = Backup + Storage = "<%= @storage_name %>" + AutoPrune = yes + Volume Retention = 30 days + Label Format = "<%= @pool_name %>-inc.${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}_${Hour:p/2/0/r}:${Minute:p/2/0/r}" + Volume Use Duration = 23h + Maximum Volume Jobs = 1 + Maximum Volume Bytes = 500G + Action On Purge = Truncate + Recycle = yes + RecyclePool = "poolgraveyard-<%= @pool_name %>" +} + +Pool { + Name = "poolgraveyard-<%=@pool_name %>" + Pool Type = Backup + Storage = "<%= @storage_name %>" + Recycle = yes + RecyclePool = "poolgraveyard-<%= @pool_name %>" +} diff --git a/modules/bacula/templates/director/dir-per-client.erb b/modules/bacula/templates/director/dir-per-client.erb index 213bfe2e1..713bc3e13 100644 --- a/modules/bacula/templates/director/dir-per-client.erb +++ b/modules/bacula/templates/director/dir-per-client.erb @@ -7,11 +7,11 @@ Job { Name = "<%= @client %>" JobDefs = "Standardbackup" - Client = <%= @bacula_client_name %> + Client = <%= @client_name %> - Pool = "poolfull-<%= @bacula_pool_name %>-<%= @client %>" - Differential Backup Pool = "pooldiff-<%= @bacula_pool_name %>-<%= @client %>" - Incremental Backup Pool = "poolinc-<%= @bacula_pool_name %>-<%= @client %>" + Pool = "poolfull-<%= @pool_name %>" + Differential Backup Pool = "pooldiff-<%= @pool_name %>" + Incremental Backup Pool = "poolinc-<%= @pool_name %>" Reschedule On Error = yes Reschedule Interval = 4 hours @@ -23,11 +23,11 @@ Job { # Client (File Services) to backup Client { - Name = <%= @bacula_client_name %> + Name = <%= @client_name %> Address = <%= @client %> FDPort = <%= @port_fd %> Catalog = MyCatalog - Password = "<%= @bacula_client_secret %>" + Password = "<%= @client_secret %>" File Retention = 30 days Job Retention = 3 months AutoPrune = yes @@ -40,79 +40,3 @@ Client { TLS Key = "<%= @bacula_ssl_client_key %>" } -######################################################################## -# Storage config # -######################################################################## - -Storage { - Name = "<%= @bacula_filestor_name %>-<%= @client %>" - Address = <%= @bacula_storage_address %> - SDPort = <%= @bacula_storage_port %> - Password = "<%= @bacula_storage_secret %>" - Device = "<%= @bacula_filestor_device %>-<%= @client %>" - Media Type = "<%= @bacula_filestor_name %>-<%= @client %>" - Maximum Concurrent Jobs = 10 - - TLS Enable = yes - TLS Require = yes - TLS CA Certificate File = "<%= @bacula_ca_path %>" - # This is a client certificate, used by the director to connect to the storage daemon - TLS Certificate = "<%= @bacula_ssl_client_cert %>" - TLS Key = "<%= @bacula_ssl_client_key %>" -} - -######################################################################## -# Pool definition # -######################################################################## -Pool { - Name = "poolfull-<%=@bacula_pool_name%>-<%= @client %>" - Pool Type = Backup - Storage = "<%=@bacula_filestor_name%>-<%= @client %>" - AutoPrune = yes - Volume Retention = 100 days - Label Format = "<%= @bacula_pool_name %>-full-<%= @client %>.${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}_${Hour:p/2/0/r}:${Minute:p/2/0/r}" - Volume Use Duration = 23h - Maximum Volume Jobs = 1 - Maximum Volume Bytes = 500G - Action On Purge = Truncate - Recycle = yes - RecyclePool = "poolgraveyard-<%=@bacula_pool_name%>-<%= @client %>" -} - -Pool { - Name = "pooldiff-<%=@bacula_pool_name%>-<%= @client %>" - Pool Type = Backup - Storage = "<%=@bacula_filestor_name%>-<%= @client %>" - AutoPrune = yes - Volume Retention = 50 days - Label Format = "<%= @bacula_pool_name %>-diff-<%= @client %>.${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}_${Hour:p/2/0/r}:${Minute:p/2/0/r}" - Volume Use Duration = 23h - Maximum Volume Jobs = 1 - Maximum Volume Bytes = 500G - Action On Purge = Truncate - Recycle = yes - RecyclePool = "poolgraveyard-<%=@bacula_pool_name%>-<%= @client %>" -} - -Pool { - Name = "poolinc-<%=@bacula_pool_name%>-<%= @client %>" - Pool Type = Backup - Storage = "<%=@bacula_filestor_name%>-<%= @client %>" - AutoPrune = yes - Volume Retention = 30 days - Label Format = "<%= @bacula_pool_name %>-inc-<%= @client %>.${Year}-${Month:p/2/0/r}-${Day:p/2/0/r}_${Hour:p/2/0/r}:${Minute:p/2/0/r}" - Volume Use Duration = 23h - Maximum Volume Jobs = 1 - Maximum Volume Bytes = 500G - Action On Purge = Truncate - Recycle = yes - RecyclePool = "poolgraveyard-<%=@bacula_pool_name%>-<%= @client %>" -} - -Pool { - Name = "poolgraveyard-<%=@bacula_pool_name%>-<%= @client %>" - Pool Type = Backup - Storage = "<%=@bacula_filestor_name%>-<%= @client %>" - Recycle = yes - RecyclePool = "poolgraveyard-<%=@bacula_pool_name%>-<%= @client %>" -} -- 2.20.1