From 392e83188b61b78c66a075668dc15be42fafbd11 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 13 Mar 2023 16:20:03 +0100 Subject: [PATCH] Fix backups for new multionode dbs --- manifests/script.pp | 51 ++++++++++++++----- manifests/scriptreceiver.pp | 2 +- templates/script/backupmultinodedb.erb.sh | 24 +++++++++ .../scriptreceiver/backup_multinode_db.erb.sh | 5 ++ 4 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 templates/script/backupmultinodedb.erb.sh create mode 100644 templates/scriptreceiver/backup_multinode_db.erb.sh diff --git a/manifests/script.pp b/manifests/script.pp index 493beb2..491d7a0 100644 --- a/manifests/script.pp +++ b/manifests/script.pp @@ -312,14 +312,33 @@ class sunetdrive::script ( ok_criteria => ['exit_status=0','max_age=2d'], warn_criteria => ['exit_status=1','max_age=3d'], } + file { '/root/tasks/backupsinglenodedb.sh': + ensure => file, + content => template('sunetdrive/script/backupsinglenodedb.erb.sh'), + owner => 'root', + group => 'root', + mode => '0700', + } } - file { '/root/tasks/backupsinglenodedb.sh': - ensure => file, - content => template('sunetdrive/script/backupsinglenodedb.erb.sh'), - owner => 'root', - group => 'root', - mode => '0700', + else { + file { '/root/tasks/backupsinglenodedb.sh': + ensure => absent, + } + file { '/root/tasks/backupmultinodedb.sh': + ensure => file, + content => template('sunetdrive/script/backupmultinodedb.erb.sh'), + owner => 'root', + group => 'root', + mode => '0700', + } + sunet::scriptherder::cronjob { "backupmultinodedb": + cmd => "/root/tasks/backupmultinodedb.sh", + hour => '2', + minute => '0', + ok_criteria => ['exit_status=0','max_age=2d'], + warn_criteria => ['exit_status=1','max_age=3d'], + } } $singlenodes.each | $singlenode| { $multinode = hiera_hash('multinode_mapping')[$singlenode]['server'] @@ -332,14 +351,14 @@ class sunetdrive::script ( ok_criteria => ['exit_status=0','max_age=30m'], warn_criteria => ['exit_status=1', 'max_age=60m'], } - sunet::scriptherder::cronjob { "backup${singlenode}db": - cmd => "/root/tasks/backupsinglenodedb.sh ${multinodeserver} ${singlenode}", - hour => '2', - minute => '0', - ok_criteria => ['exit_status=0','max_age=2d'], - warn_criteria => ['exit_status=1','max_age=3d'], - } if $environment == 'prod' { + sunet::scriptherder::cronjob { "backup${singlenode}db": + cmd => "/root/tasks/backupsinglenodedb.sh ${multinodeserver} ${singlenode}", + hour => '2', + minute => '0', + ok_criteria => ['exit_status=0','max_age=2d'], + warn_criteria => ['exit_status=1','max_age=3d'], + } sunet::scriptherder::cronjob { "statistics${singlenode}": cmd => "/root/tasks/usage.sh ${singlenode} ${multinodeserver}", hour => '2', @@ -348,6 +367,12 @@ class sunetdrive::script ( warn_criteria => ['exit_status=1','max_age=3d'], } } + else { + sunet::scriptherder::cronjob { "backup${singlenode}db": + ensure => absent, + cmd => 'true', + } + } unless $singlenode in ['mau', 'uu'] { sunet::scriptherder::cronjob { "make${singlenode}buckets": cmd => "/root/tasks/makebuckets.sh ${multinodeserver} ${nccontainer} ${singlenode}-${environment}", diff --git a/manifests/scriptreceiver.pp b/manifests/scriptreceiver.pp index a67d760..9187dcc 100644 --- a/manifests/scriptreceiver.pp +++ b/manifests/scriptreceiver.pp @@ -4,7 +4,7 @@ class sunetdrive::scriptreceiver() sunet::system_user {'script': username => 'script', group => 'script', managehome => true, shell => '/bin/bash' } # These tasks correspond to a ${task}.erb.sh template - $tasks = ['list_users', 'list_files_for_user', 'create_bucket', 'backup_db', 'purge_backups', 'maintenancemode', 'restart_sunet_service', 'start_sentinel', 'stop_sentinel', 'makeswap'] + $tasks = ['list_users', 'list_files_for_user', 'create_bucket', 'backup_db', 'purge_backups', 'maintenancemode', 'restart_sunet_service', 'start_sentinel', 'stop_sentinel', 'makeswap', 'backup_multinode_db'] $environment = sunetdrive::get_environment() $config = hiera_hash($environment) diff --git a/templates/script/backupmultinodedb.erb.sh b/templates/script/backupmultinodedb.erb.sh new file mode 100644 index 0000000..91de93c --- /dev/null +++ b/templates/script/backupmultinodedb.erb.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Backup all databases +# We sleep a deterministic amount of time, which will be between 0 an 128 m and allways the same within +# a specific host, but will differ between hosts +sleep $((16#$(ip a | grep "link/ether" | head -1 | awk -F ':' '{print $6}' | awk '{print $1}') / 2))m +number_of_full_to_keep=7 +backup="multinode-db1.$(fqdn -d)" +remote_backup_dir="/etc/mariadb/backups" +backup_dir="/opt/backups" +bucket="db-backups-multinode" +mirror="common-<%= @environment %>-mirror" +echo "Backing up all databases for for multinode customer" +ssh "${backup}" "sudo /home/script/bin/backup_multinode_db.sh" +echo "Cleaning up old backups for ${backup}" +ssh ${backup} "sudo /home/script/bin/purge_backups.sh ${remote_backup_dir}" +echo "Copying backups here" +mkdir -p ${backup_dir} +scp "script@${backup}:${remote_backup_dir}/$(date +%Y/%m/%d)/*.gz" "${backup_dir}" +echo "Copying backups to remote bucket" +rclone mkdir "${mirror}:${bucket}" +duplicity --full-if-older-than 1M --tempdir /mnt --archive-dir /mnt --no-encryption "${backup_dir}" "rclone://${mirror}:/${bucket}" +duplicity remove-all-but-n-full "${number_of_full_to_keep}" --tempdir /mnt --archive-dir /mnt --force "rclone://${mirror}:/${bucket}" +echo "cleaning up" +rm -r "${backup_dir}" diff --git a/templates/scriptreceiver/backup_multinode_db.erb.sh b/templates/scriptreceiver/backup_multinode_db.erb.sh new file mode 100644 index 0000000..f52020b --- /dev/null +++ b/templates/scriptreceiver/backup_multinode_db.erb.sh @@ -0,0 +1,5 @@ +#!/bin/bash +docker exec mariadb_db_1 /scripts/run_manual_backup_dump.sh +backupdir=/etc/mariadb/backups +chmod 755 ${backupdir} +chown -R script:root ${backupdir}