From d2ced8d1cd56769a35a6f2967f3cf9722ef52aff Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Mon, 25 Sep 2023 09:34:19 +0200 Subject: [PATCH] Backups: This script does not seem used and is confusing --- templates/script/backup-buckets-v2.erb.sh | 86 ----------------------- 1 file changed, 86 deletions(-) delete mode 100644 templates/script/backup-buckets-v2.erb.sh diff --git a/templates/script/backup-buckets-v2.erb.sh b/templates/script/backup-buckets-v2.erb.sh deleted file mode 100644 index 1979b12..0000000 --- a/templates/script/backup-buckets-v2.erb.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# Backup all buckets -# We sleep a deterministic amount of time, which will be between 0 an 128 m and allways the same within -# a specific host, but will differ between hosts -sleep $((16#$(ip a | grep "link/ether" | head -1 | awk -F ':' '{print $6}' | awk '{print $1}') / 2))m -number_of_full_to_keep="<%= @full_backup_retention %>" -fork_limit=30 #in GB, if bigger than this number, we fork the backup to it's own process -split_limit=1000 #in GB, if bigger than this number, we fork backup of each directory to it's own process - -declare -a projects=("<%= @primary_project %> <%= @mirror_project %>") -#<% @assigned_projects.each do |project| %> -projects+=("<%= project['project'] %> <%= project['mirror_project'] %>") -#<% end %> - -function do_huge_backup { - local project="${1}" - local mirror="${2}" - local bucket="${3}" - declare -a directories - declare -a empty - for dir in $(rclone lsd ${project}:${bucket} | awk '{print $NF}'); do - directories+=("${dir}") - mountpoint="/opt/backupmounts/${bucket}-${dir}" - do_backup ${project} ${mirror} ${bucket} ${mountpoint} ${dir} ${empty} & - done - mountpoint="/opt/backupmounts/${bucket}" - do_backup ${project} ${mirror} ${bucket} ${mountpoint} none ${directories[@]} & - -} - -function do_backup { - local project="${1}" - shift - local mirror="${1}" - shift - local bucket="${1}" - shift - local mountpoint="${1}" - shift - local dire="${1}" - shift - declare -a exclude - exclude=( "${@}" ) - suffix="" - opts="" - if [[ "${dire}" != "none" ]]; then - suffix="/${dire}" - fi - if ((${#exclude[@]})); then - for dir in "${exclude[@]}"; do - opts="${opts} --exclude /${dir}" - done - fi - local mirrorbucket="${bucket}-mirror" - mkdir -p ${mountpoint} - rclone mount ${project}:${bucket}${suffix} ${mountpoint}/ --daemon --allow-other - rclone mkdir ${mirror}:${mirrorbucket}${suffix} - duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt ${opts} \ - --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket}${suffix} - umount ${mountpoint} - rmdir ${mountpoint} - # Clean up - duplicity remove-all-but-n-full ${number_of_full_to_keep} --tempdir /mnt --archive-dir /mnt \ - --force rclone://${mirror}:/${mirrorbucket}${suffix} -} - -for entry in "${projects[@]}"; do - project=$(echo ${entry} | awk '{print $1}') - mirror=$(echo ${entry} | awk '{print $2}') - declare -a empty - for bucket in $(rclone lsd ${project}:/ | awk '{print $5}'); do - size=$(rclone size --json ${project}:${bucket} | jq -r '.bytes') - mirrorbucket="${bucket}-mirror" - mountpoint="/opt/backupmounts/${bucket}" - # If bucket is above ${split_limit} we fork and do backup per directory - if [[ ${size} -gt $((${split_limit} * 1000000000)) ]]; then - do_huge_backup ${project} ${mirror} ${bucket} & - # If bucket is above ${fork_limit} we fork and do backup for bucket - elif [[ ${size} -gt $((${fork_limit} * 1000000000)) ]]; then - do_backup ${project} ${mirror} ${bucket} ${mountpoint} none ${empty} & - else - # If bucket is below ${fork_limit} we do not fork and do backup for bucket - do_backup ${project} ${mirror} ${bucket} ${mountpoint} none ${empty} - fi - done -done