From 4d6e6f67af3ef04fff39a17916287562feba4988 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Tue, 25 Jun 2024 11:19:53 +0200 Subject: [PATCH 1/8] Use correect variable --- templates/script/backup-all-buckets.erb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/script/backup-all-buckets.erb.sh b/templates/script/backup-all-buckets.erb.sh index d808c42..cf19cdc 100644 --- a/templates/script/backup-all-buckets.erb.sh +++ b/templates/script/backup-all-buckets.erb.sh @@ -25,9 +25,9 @@ projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= singl #<% end %> #<% end %> #<% else %> -declare -a projects=("<%= @primary_project %> <%= @mirror_project %> <%= customer %>") +declare -a projects=("<%= @primary_project %> <%= @mirror_project %> ${customer}") #<% @assigned_projects.each do |project| %> -projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= customer %>") +projects+=("<%= project['project'] %> <%= project['mirror_project'] %> ${customer}") #<% end %> #<% end %> From 1dea001ed995ae1eade6adfd769faa0fd8bbfd63 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Tue, 25 Jun 2024 18:09:13 +0200 Subject: [PATCH 2/8] No ports for mode host --- templates/lookup/docker-compose_lookup.yml.erb | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/lookup/docker-compose_lookup.yml.erb b/templates/lookup/docker-compose_lookup.yml.erb index 59fc38b..0fbf94d 100644 --- a/templates/lookup/docker-compose_lookup.yml.erb +++ b/templates/lookup/docker-compose_lookup.yml.erb @@ -13,7 +13,5 @@ services: - 89.46.20.75 - 89.46.21.29 - 89.32.32.32 - ports: - - 443:443 command: apachectl -D FOREGROUND tty: true From db92a5d25547e239a40e7fce76afe2b3e3ec2041 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 26 Jun 2024 09:14:59 +0200 Subject: [PATCH 3/8] Clean up logic --- templates/script/backup-all-buckets.erb.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/templates/script/backup-all-buckets.erb.sh b/templates/script/backup-all-buckets.erb.sh index cf19cdc..c15b605 100644 --- a/templates/script/backup-all-buckets.erb.sh +++ b/templates/script/backup-all-buckets.erb.sh @@ -46,16 +46,20 @@ function do_backup { ps aux | grep duplicity | grep "[^a-zA-Z]${bucket}" > /dev/null local oktorun=$? # 1 == this bucket has no other bakup process in progress mkdir -p ${mountpoint} - [[ ${oktorun} -ne 0 ]] && rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h - rclone mkdir ${mirror}:${mirrorbucket} - [[ ${oktorun} -ne 0 ]] && duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \ - --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} && \ - [[ -n "${extra_backup_jobs[${customer}]}" ]] && [[ -f ${extra_backup_jobs[${customer} ]] && ${extra_backup_jobs[${customer}]} - umount ${mountpoint} - rmdir ${mountpoint} - # Clean up - [ ${oktorun} -ne 0 ] && duplicity remove-all-but-n-full ${number_of_full_to_keep} --tempdir /mnt --archive-dir /mnt \ + if [[ ${oktorun} -ne 0 ]]; then + rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h + rclone mkdir ${mirror}:${mirrorbucket} + duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \ + --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} + if [[ -n "${extra_backup_jobs[${customer}]}" ]] && [[ -f ${extra_backup_jobs[${customer} ]]; then + ${extra_backup_jobs[${customer}]} + fi + umount ${mountpoint} + rmdir ${mountpoint} + # Clean up + duplicity remove-all-but-n-full ${number_of_full_to_keep} --tempdir /mnt --archive-dir /mnt \ --force rclone://${mirror}:/${mirrorbucket} + fi } for entry in "${projects[@]}"; do From 42831948e1002e5820462261dcedd268d3c953b4 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 26 Jun 2024 09:40:36 +0200 Subject: [PATCH 4/8] Fix syntax error --- templates/script/backup-all-buckets.erb.sh | 56 +++++++++++----------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/templates/script/backup-all-buckets.erb.sh b/templates/script/backup-all-buckets.erb.sh index c15b605..96ccd29 100644 --- a/templates/script/backup-all-buckets.erb.sh +++ b/templates/script/backup-all-buckets.erb.sh @@ -15,7 +15,7 @@ extra_backup_jobs["<%= client %>"]="<%= job %>" #<% if @location.start_with?('common') %> declare -a sixmonths=('mau') if [[ " ${sixmonths[*]} " =~ " ${customer} " ]]; then - number_of_full_to_keep=6 + number_of_full_to_keep=6 fi declare -a projects #<% @singlenodes.each do |singlenode| %> @@ -37,47 +37,45 @@ fi function do_backup { - local project="${1}" - local mirror="${2}" - local bucket="${3}" - local customer="${4}" - local mirrorbucket="${bucket}-mirror" - local mountpoint="/opt/backupmounts/${bucket}" - ps aux | grep duplicity | grep "[^a-zA-Z]${bucket}" > /dev/null - local oktorun=$? # 1 == this bucket has no other bakup process in progress - mkdir -p ${mountpoint} - if [[ ${oktorun} -ne 0 ]]; then + local project="${1}" + local mirror="${2}" + local bucket="${3}" + local customer="${4}" + local mirrorbucket="${bucket}-mirror" + local mountpoint="/opt/backupmounts/${bucket}" + ps aux | grep duplicity | grep "[^a-zA-Z]${bucket}" > /dev/null + local oktorun=$? # 1 == this bucket has no other bakup process in progress + if [[ ${oktorun} -ne 0 ]]; then + mkdir -p ${mountpoint} rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h rclone mkdir ${mirror}:${mirrorbucket} - duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \ - --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} - if [[ -n "${extra_backup_jobs[${customer}]}" ]] && [[ -f ${extra_backup_jobs[${customer} ]]; then + duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} + if [[ "${extra_backup_jobs[${customer}]:+none}" != "none" ]] && [[ -f "${extra_backup_jobs[${customer}]}" ]]; then ${extra_backup_jobs[${customer}]} fi umount ${mountpoint} rmdir ${mountpoint} - # Clean up - duplicity remove-all-but-n-full ${number_of_full_to_keep} --tempdir /mnt --archive-dir /mnt \ - --force rclone://${mirror}:/${mirrorbucket} + # Clean up + duplicity remove-all-but-n-full ${number_of_full_to_keep} --tempdir /mnt --archive-dir /mnt --force rclone://${mirror}:/${mirrorbucket} fi } for entry in "${projects[@]}"; do - project=$(echo ${entry} | awk '{print $1}') - mirror=$(echo ${entry} | awk '{print $2}') - customer=$(echo ${entry} | awk '{print $3}') - for bucket in $(rclone lsd ${project}:/ | awk '{print $5}'); do - maybesize=$(timeout 30s rclone size --json ${project}:${bucket}) + project=$(echo ${entry} | awk '{print $1}') + mirror=$(echo ${entry} | awk '{print $2}') + customer=$(echo ${entry} | awk '{print $3}') + for bucket in $(rclone lsd ${project}:/ | awk '{print $5}'); do + maybesize=$(timeout 30s rclone size --json ${project}:${bucket}) if [[ ${?} -eq 124 ]]; then size=$((${fork_limit} * 1000000001)) else size=$(echo ${maybesize} | jq -r '.bytes' ) fi - # If bucket is above 50 GB we fork - if [[ ${size} -gt $((${fork_limit} * 1000000000)) ]]; then - do_backup ${project} ${mirror} ${bucket} ${customer} & - else - do_backup ${project} ${mirror} ${bucket} ${customer} - fi - done + # If bucket is above 50 GB we fork + if [[ ${size} -gt $((${fork_limit} * 1000000000)) ]]; then + do_backup ${project} ${mirror} ${bucket} ${customer} & + else + do_backup ${project} ${mirror} ${bucket} ${customer} + fi + done done From bac25d27c8b389c4e3a81f0930cacc8b5741f0ed Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 26 Jun 2024 09:58:18 +0200 Subject: [PATCH 5/8] Fix bug in script --- manifests/script.pp | 2 +- .../create_folders_in_fullnode_buckets.erb.sh | 46 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/manifests/script.pp b/manifests/script.pp index f4a9e84..b70c78d 100644 --- a/manifests/script.pp +++ b/manifests/script.pp @@ -250,7 +250,7 @@ class sunetdrive::script ( if $customer in ['extern', 'gih', 'hkr', 'suni', 'common', 'su', 'lnu'] { sunet::scriptherder::cronjob { 'makebuckets': ensure => absent, - cmd => 'bin/true', + cmd => 'bin/true', } } else { sunet::scriptherder::cronjob { 'makebuckets': diff --git a/templates/script/create_folders_in_fullnode_buckets.erb.sh b/templates/script/create_folders_in_fullnode_buckets.erb.sh index 8cfe792..526ce58 100644 --- a/templates/script/create_folders_in_fullnode_buckets.erb.sh +++ b/templates/script/create_folders_in_fullnode_buckets.erb.sh @@ -7,16 +7,16 @@ include_userbuckets="<%= @include_userbuckets %>" container="nextcloud_app_1" yq="/usr/local/bin/yq" if ! [[ -x ${yq} ]]; then - pip install yq + pip install yq fi declare -a directories if [[ -n ${1} ]]; then - directories=("${@}") + directories=("${@}") else - directories+=("Arbetsmaterial") - directories+=("Bevarande") - directories+=("Gallringsbart") + directories+=("Arbetsmaterial") + directories+=("Bevarande") + directories+=("Gallringsbart") fi olddir="${PWD}" @@ -25,31 +25,31 @@ dirty=0 primary='' declare -a users=( 'admin' ) cd "${tempdir}" || echo "Could not cd to tempdir" -declare -a projects=( "${yq}" -r '.project_mapping.'"${customer}"'.'"${environment}"'.assigned | "\(.[].project)"' /etc/hiera/data/common.yaml ) +declare -a projects=( $("${yq}" -r '.project_mapping.'"${customer}"'.'"${environment}"'.assigned | "\(.[].project)"' /etc/hiera/data/common.yaml) ) if [[ "${include_userbuckets}" == "true" ]]; then - primary=$("${yq}" -r '.project_mapping.'"${customer}"'.'"${environment}"'.primary_project' /etc/hiera/data/common.yaml) - projects+=( "${primary}" ) + primary=$("${yq}" -r '.project_mapping.'"${customer}"'.'"${environment}"'.primary_project' /etc/hiera/data/common.yaml) + projects+=( "${primary}" ) fi for project in "${projects[@]}"; do - for bucket in $(rclone lsd "${project}:" | awk '{print $NF}' | grep -E -v '^primary'); do - count=$(rclone size --json "${project}:${bucket}" | jq -r .count) - if [[ ${count} -gt 0 ]]; then - echo "Skipping ${project}:${bucket} because it has stuff in it already" - continue - fi - for directory in "${directories[@]}"; do - dirty=1 + for bucket in $(rclone lsd "${project}:" | awk '{print $NF}' | grep -E -v '^primary'); do + count=$(rclone size --json "${project}:${bucket}" | jq -r .count) + if [[ ${count} -gt 0 ]]; then + echo "Skipping ${project}:${bucket} because it has stuff in it already" + continue + fi + for directory in "${directories[@]}"; do + dirty=1 if [[ -n ${primary} ]] && [[ ${project} == "${primary}" ]] ; then user=$(echo "${bucket}" | awk -F '-' '{print $1}') users+=( "${user}@${eppn_suffix}" ) fi - echo "Creating ${project}:${bucket}/${directory} because it looks nice and empty" - temp="README.md" - echo "**${directory}**" >"${temp}" - echo "Var god lämna kvar denna fil/Please leave this file" >>"${temp}" - rclone --no-traverse move "${temp}" "${project}:${bucket}/${directory}" - done - done + echo "Creating ${project}:${bucket}/${directory} because it looks nice and empty" + temp="README.md" + echo "**${directory}**" >"${temp}" + echo "Var god lämna kvar denna fil/Please leave this file" >>"${temp}" + rclone --no-traverse move "${temp}" "${project}:${bucket}/${directory}" + done + done done cd "${olddir}" || echo "could not cd to home dir" rmdir "${tempdir}" From 7f7d38c87e222c3cd000f7d6803a2c46b4df1bab Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 26 Jun 2024 10:19:53 +0200 Subject: [PATCH 6/8] Switch logic --- templates/script/backup-all-buckets.erb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/script/backup-all-buckets.erb.sh b/templates/script/backup-all-buckets.erb.sh index 96ccd29..853b772 100644 --- a/templates/script/backup-all-buckets.erb.sh +++ b/templates/script/backup-all-buckets.erb.sh @@ -50,7 +50,7 @@ function do_backup { rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h rclone mkdir ${mirror}:${mirrorbucket} duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} - if [[ "${extra_backup_jobs[${customer}]:+none}" != "none" ]] && [[ -f "${extra_backup_jobs[${customer}]}" ]]; then + if [[ "${extra_backup_jobs[${customer}]:+found}" == "found" ]] && [[ -f "${extra_backup_jobs[${customer}]}" ]]; then ${extra_backup_jobs[${customer}]} fi umount ${mountpoint} From 4cf8cf3adbdccd28fcce5fbff5e8f7009fd775c7 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 26 Jun 2024 12:48:50 +0200 Subject: [PATCH 7/8] Add backup script for hb --- manifests/script.pp | 11 +++++++++++ templates/script/backup-hb.erb.sh | 6 ++++++ 2 files changed, 17 insertions(+) create mode 100755 templates/script/backup-hb.erb.sh diff --git a/manifests/script.pp b/manifests/script.pp index b70c78d..fdabf30 100644 --- a/manifests/script.pp +++ b/manifests/script.pp @@ -97,6 +97,10 @@ class sunetdrive::script ( unless => "python3 -m pip list | grep drive-utils | grep ${drive_version}", require => Package['python3'], } + file { '/opt/backups/scripts': + ensure => directory, + mode => '0700' + } file { '/root/.ssh/': ensure => directory, mode => '0700', @@ -335,6 +339,13 @@ class sunetdrive::script ( group => 'root', mode => '0700', } + file { '/opt/backups/scripts/hb.sh': + ensure => file, + content => template('sunetdrive/script/backup-hb.erb.sh'), + owner => 'root', + group => 'root', + mode => '0700', + } sunet::scriptherder::cronjob { 'backupmultinodedb': cmd => '/root/tasks/backupmultinodedb.sh', hour => '2', diff --git a/templates/script/backup-hb.erb.sh b/templates/script/backup-hb.erb.sh new file mode 100755 index 0000000..706b589 --- /dev/null +++ b/templates/script/backup-hb.erb.sh @@ -0,0 +1,6 @@ +#!/bin/bash +environ="prod" +if [[ $(hostname -d) == "drive.test.sunet.se" ]]; then + environ="test" +fi +rsync -e "ssh -i ${HOME}/.ssh/id_script" -avz /opt/backupmounts/ "sd-${environ}@sd-${environ}-backup.hb.se:~/sd-${environ}" From d3530696602fccbd2330414283360dad46e28149 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Thu, 27 Jun 2024 09:31:37 +0200 Subject: [PATCH 8/8] Make a standard about which params a script gets. project="${1}" mirror="${2}" bucket="${3}" customer="${4}" environment="${5}" --- templates/script/backup-all-buckets.erb.sh | 4 +++- templates/script/backup-hb.erb.sh | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/templates/script/backup-all-buckets.erb.sh b/templates/script/backup-all-buckets.erb.sh index 853b772..c5179df 100644 --- a/templates/script/backup-all-buckets.erb.sh +++ b/templates/script/backup-all-buckets.erb.sh @@ -6,7 +6,9 @@ sleep $((16#$(ip a | grep "link/ether" | head -1 | awk -F ':' '{print $6}' | awk number_of_full_to_keep='<%= @full_backup_retention %>' fork_limit=30 #in GB, if bigger than this number, we fork the backup to it's own process customer="<%= @customer %>" +environment="<%= @environment %>" declare -A extra_backup_jobs + #<% if @extra_backup_jobs.any? %> #<% @extra_backup_jobs.each do |client, job| %> extra_backup_jobs["<%= client %>"]="<%= job %>" @@ -51,7 +53,7 @@ function do_backup { rclone mkdir ${mirror}:${mirrorbucket} duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} if [[ "${extra_backup_jobs[${customer}]:+found}" == "found" ]] && [[ -f "${extra_backup_jobs[${customer}]}" ]]; then - ${extra_backup_jobs[${customer}]} + ${extra_backup_jobs[${customer}]} ${project} ${mirror} ${bucket} ${customer} ${environment} fi umount ${mountpoint} rmdir ${mountpoint} diff --git a/templates/script/backup-hb.erb.sh b/templates/script/backup-hb.erb.sh index 706b589..93cf169 100755 --- a/templates/script/backup-hb.erb.sh +++ b/templates/script/backup-hb.erb.sh @@ -1,6 +1,8 @@ #!/bin/bash -environ="prod" -if [[ $(hostname -d) == "drive.test.sunet.se" ]]; then - environ="test" -fi -rsync -e "ssh -i ${HOME}/.ssh/id_script" -avz /opt/backupmounts/ "sd-${environ}@sd-${environ}-backup.hb.se:~/sd-${environ}" +project="${1}" +mirror="${2}" +bucket="${3}" +customer="${4}" +environment="${5}" + +rsync -e "ssh -i ${HOME}/.ssh/id_script" -avz "/opt/backupmounts/${bucket}" "sd-${environment}@sd-${environment}-backup.hb.se:~/sd-${environment}/${bucket}"