Fix backups for new multionode dbs

manifests/script.pp

@@ -312,8 +312,6 @@ class sunetdrive::script (
         ok_criteria   => ['exit_status=0','max_age=2d'],
         warn_criteria => ['exit_status=1','max_age=3d'],
       }
-
-    }
       file { '/root/tasks/backupsinglenodedb.sh':
         ensure  => file,
         content => template('sunetdrive/script/backupsinglenodedb.erb.sh'),
@@ -321,6 +319,27 @@ class sunetdrive::script (
         group   => 'root',
         mode    => '0700',
       }
+
+    }
+    else {
+      file { '/root/tasks/backupsinglenodedb.sh':
+        ensure  => absent,
+      }
+      file { '/root/tasks/backupmultinodedb.sh':
+        ensure  => file,
+        content => template('sunetdrive/script/backupmultinodedb.erb.sh'),
+        owner   => 'root',
+        group   => 'root',
+        mode    => '0700',
+      }
+      sunet::scriptherder::cronjob { "backupmultinodedb":
+        cmd           => "/root/tasks/backupmultinodedb.sh",
+        hour          => '2',
+        minute        => '0',
+        ok_criteria   => ['exit_status=0','max_age=2d'],
+        warn_criteria => ['exit_status=1','max_age=3d'],
+      }
+    }
     $singlenodes.each | $singlenode| {
       $multinode = hiera_hash('multinode_mapping')[$singlenode]['server']
       $multinodeserver = "${multinode}.${site_name}"
@@ -332,6 +351,7 @@ class sunetdrive::script (
         ok_criteria   => ['exit_status=0','max_age=30m'],
         warn_criteria => ['exit_status=1', 'max_age=60m'],
       }
+      if $environment == 'prod' {
         sunet::scriptherder::cronjob { "backup${singlenode}db":
           cmd           => "/root/tasks/backupsinglenodedb.sh ${multinodeserver} ${singlenode}",
           hour          => '2',
@@ -339,7 +359,6 @@ class sunetdrive::script (
           ok_criteria   => ['exit_status=0','max_age=2d'],
           warn_criteria => ['exit_status=1','max_age=3d'],
         }
-      if $environment == 'prod' {
         sunet::scriptherder::cronjob { "statistics${singlenode}":
           cmd           => "/root/tasks/usage.sh ${singlenode} ${multinodeserver}",
           hour          => '2',
@@ -348,6 +367,12 @@ class sunetdrive::script (
           warn_criteria => ['exit_status=1','max_age=3d'],
         }
       }
+      else {
+        sunet::scriptherder::cronjob { "backup${singlenode}db":
+          ensure        => absent,
+          cmd           => 'true',
+        }
+      }
       unless $singlenode in ['mau', 'uu'] {
         sunet::scriptherder::cronjob { "make${singlenode}buckets":
           cmd           => "/root/tasks/makebuckets.sh ${multinodeserver} ${nccontainer} ${singlenode}-${environment}",

manifests/scriptreceiver.pp

@@ -4,7 +4,7 @@ class sunetdrive::scriptreceiver()
   sunet::system_user {'script': username => 'script', group => 'script', managehome => true, shell => '/bin/bash' }
 
   # These tasks correspond to a ${task}.erb.sh template
-  $tasks = ['list_users', 'list_files_for_user', 'create_bucket', 'backup_db', 'purge_backups', 'maintenancemode', 'restart_sunet_service', 'start_sentinel', 'stop_sentinel', 'makeswap']
+  $tasks = ['list_users', 'list_files_for_user', 'create_bucket', 'backup_db', 'purge_backups', 'maintenancemode', 'restart_sunet_service', 'start_sentinel', 'stop_sentinel', 'makeswap', 'backup_multinode_db']
 
   $environment = sunetdrive::get_environment()
   $config = hiera_hash($environment)

templates/script/backupmultinodedb.erb.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Backup all databases
+# We sleep a deterministic amount of time, which will be between 0 an 128 m and allways the same within
+# a specific host, but will differ between hosts
+sleep $((16#$(ip a | grep "link/ether" | head -1 | awk -F ':' '{print $6}' | awk '{print $1}') / 2))m
+number_of_full_to_keep=7
+backup="multinode-db1.$(fqdn -d)"
+remote_backup_dir="/etc/mariadb/backups"
+backup_dir="/opt/backups"
+bucket="db-backups-multinode"
+mirror="common-<%= @environment %>-mirror"
+echo "Backing up all databases for for multinode customer"
+ssh "${backup}" "sudo /home/script/bin/backup_multinode_db.sh"
+echo "Cleaning up old backups for ${backup}"
+ssh ${backup} "sudo /home/script/bin/purge_backups.sh ${remote_backup_dir}"
+echo "Copying backups here"
+mkdir -p ${backup_dir}
+scp "script@${backup}:${remote_backup_dir}/$(date +%Y/%m/%d)/*.gz" "${backup_dir}"
+echo "Copying backups to remote bucket"
+rclone mkdir "${mirror}:${bucket}"
+duplicity --full-if-older-than 1M --tempdir /mnt --archive-dir /mnt --no-encryption "${backup_dir}" "rclone://${mirror}:/${bucket}"
+duplicity remove-all-but-n-full "${number_of_full_to_keep}" --tempdir /mnt --archive-dir /mnt --force "rclone://${mirror}:/${bucket}"
+echo "cleaning up"
+rm -r "${backup_dir}"

templates/scriptreceiver/backup_multinode_db.erb.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+docker exec mariadb_db_1 /scripts/run_manual_backup_dump.sh
+backupdir=/etc/mariadb/backups
+chmod 755 ${backupdir}
+chown -R script:root ${backupdir}