diff --git a/templates/script/backup-all-buckets.erb.sh b/templates/script/backup-all-buckets.erb.sh index 14b3b7b..a763285 100644 --- a/templates/script/backup-all-buckets.erb.sh +++ b/templates/script/backup-all-buckets.erb.sh @@ -6,7 +6,12 @@ sleep $((16#$(ip a | grep "link/ether" | head -1 | awk -F ':' '{print $6}' | awk number_of_full_to_keep='<%= @full_backup_retention %>' fork_limit=30 #in GB, if bigger than this number, we fork the backup to it's own process customer="<%= @customer %>" -declare -a extra_backup_jobs=() +declare -A extra_backup_jobs +#<% if @extra_backup_jobs.any? %> +#<% @extra_backup_jobs.each do |client, job| %> +extra_backup_jobs["<%= client %>"]="<%= job %>" +#<% end %> +#<% end %> #<% if @location.start_with?('common') %> declare -a sixmonths=('mau') if [[ " ${sixmonths[*]} " =~ " ${customer} " ]]; then @@ -14,23 +19,20 @@ if [[ " ${sixmonths[*]} " =~ " ${customer} " ]]; then fi declare -a projects #<% @singlenodes.each do |singlenode| %> -#<% if @extra_backup_jobs[singlenode] %> -extra_backup_jobs+=("<%= @extra_backup_jobs[singlenode]['script'] %>") -#<% end %> -projects+=("<%= @full_project_mapping[singlenode][@environment]['primary_project'] %> <%= @full_project_mapping[singlenode][@environment]['mirror_project'] %>") +projects+=("<%= @full_project_mapping[singlenode][@environment]['primary_project'] %> <%= @full_project_mapping[singlenode][@environment]['mirror_project'] %> <%= singlenode %>") #<% @full_project_mapping[singlenode][@environment]['assigned'].each do |project| %> -projects+=("<%= project['project'] %> <%= project['mirror_project'] %>") +projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= singlenode %>") #<% end %> #<% end %> #<% else %> -declare -a projects=("<%= @primary_project %> <%= @mirror_project %>") +declare -a projects=("<%= @primary_project %> <%= @mirror_project %> <%= customer %>") #<% @assigned_projects.each do |project| %> -projects+=("<%= project['project'] %> <%= project['mirror_project'] %>") +projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= customer %>") #<% end %> #<% end %> if [[ ${customer} == 'common' ]]; then - projects+=("<%= @location %> <%= @location %>-mirror") + projects+=("<%= @location %> <%= @location %>-mirror <%= customer %>") fi @@ -38,15 +40,17 @@ function do_backup { local project="${1}" local mirror="${2}" local bucket="${3}" + local customer="${4}" local mirrorbucket="${bucket}-mirror" local mountpoint="/opt/backupmounts/${bucket}" ps aux | grep duplicity | grep "[^a-zA-Z]${bucket}" > /dev/null local oktorun=$? # 1 == this bucket has no other bakup process in progress mkdir -p ${mountpoint} - [ ${oktorun} -ne 0 ] && rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h + [[ ${oktorun} -ne 0 ]] && rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h rclone mkdir ${mirror}:${mirrorbucket} - [ ${oktorun} -ne 0 ] && duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \ - --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} + [[ ${oktorun} -ne 0 ]] && duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \ + --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} && \ + [[ -n "${extra_backup_jobs[${customer}]}" ]] && [[ -f ${extra_backup_jobs[${customer} ]] && ${extra_backup_jobs[${customer}]} umount ${mountpoint} rmdir ${mountpoint} # Clean up @@ -57,6 +61,7 @@ function do_backup { for entry in "${projects[@]}"; do project=$(echo ${entry} | awk '{print $1}') mirror=$(echo ${entry} | awk '{print $2}') + customer=$(echo ${entry} | awk '{print $3}') for bucket in $(rclone lsd ${project}:/ | awk '{print $5}'); do maybesize=$(timeout 30s rclone size --json ${project}:${bucket}) if [[ ${?} -eq 124 ]]; then @@ -66,9 +71,9 @@ for entry in "${projects[@]}"; do fi # If bucket is above 50 GB we fork if [[ ${size} -gt $((${fork_limit} * 1000000000)) ]]; then - do_backup ${project} ${mirror} ${bucket} & + do_backup ${project} ${mirror} ${bucket} ${customer} & else - do_backup ${project} ${mirror} ${bucket} + do_backup ${project} ${mirror} ${bucket} ${customer} fi done done