Drive/sunetdrive
7
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Trigger extra backup jobs if they exist

Browse source
This commit is contained in:
Micke Nordin 2024-06-12 10:55:11 +02:00
parent 6c8a4f430b
commit ea51f0b56b
1 changed files with 19 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
templates/script/backup-all-buckets.erb.sh
View file

@ -6,7 +6,12 @@ sleep $((16#$(ip a | grep "link/ether" | head -1 | awk -F ':' '{print $6}' | awk
number_of_full_to_keep='<%= @full_backup_retention %>' number_of_full_to_keep='<%= @full_backup_retention %>'
fork_limit=30 #in GB, if bigger than this number, we fork the backup to it's own process fork_limit=30 #in GB, if bigger than this number, we fork the backup to it's own process
customer="<%= @customer %>" customer="<%= @customer %>"
declare -a extra_backup_jobs=() declare -A extra_backup_jobs
#<% if @extra_backup_jobs.any? %>
#<% @extra_backup_jobs.each do |client, job| %>
extra_backup_jobs["<%= client %>"]="<%= job %>"
#<% end %>
#<% end %>
#<% if @location.start_with?('common') %> #<% if @location.start_with?('common') %>
declare -a sixmonths=('mau') declare -a sixmonths=('mau')
if [[ " ${sixmonths[*]} " =~ " ${customer} " ]]; then if [[ " ${sixmonths[*]} " =~ " ${customer} " ]]; then
@ -14,23 +19,20 @@ if [[ " ${sixmonths[*]} " =~ " ${customer} " ]]; then
fi fi
declare -a projects declare -a projects
#<% @singlenodes.each do |singlenode| %> #<% @singlenodes.each do |singlenode| %>
#<% if @extra_backup_jobs[singlenode] %> projects+=("<%= @full_project_mapping[singlenode][@environment]['primary_project'] %> <%= @full_project_mapping[singlenode][@environment]['mirror_project'] %> <%= singlenode %>")
extra_backup_jobs+=("<%= @extra_backup_jobs[singlenode]['script'] %>")
#<% end %>
projects+=("<%= @full_project_mapping[singlenode][@environment]['primary_project'] %> <%= @full_project_mapping[singlenode][@environment]['mirror_project'] %>")
#<% @full_project_mapping[singlenode][@environment]['assigned'].each do |project| %> #<% @full_project_mapping[singlenode][@environment]['assigned'].each do |project| %>
projects+=("<%= project['project'] %> <%= project['mirror_project'] %>") projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= singlenode %>")
#<% end %> #<% end %>
#<% end %> #<% end %>
#<% else %> #<% else %>
declare -a projects=("<%= @primary_project %> <%= @mirror_project %>") declare -a projects=("<%= @primary_project %> <%= @mirror_project %> <%= customer %>")
#<% @assigned_projects.each do |project| %> #<% @assigned_projects.each do |project| %>
projects+=("<%= project['project'] %> <%= project['mirror_project'] %>") projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= customer %>")
#<% end %> #<% end %>
#<% end %> #<% end %>
if [[ ${customer} == 'common' ]]; then if [[ ${customer} == 'common' ]]; then
projects+=("<%= @location %> <%= @location %>-mirror") projects+=("<%= @location %> <%= @location %>-mirror <%= customer %>")
fi fi
@ -38,15 +40,17 @@ function do_backup {
local project="${1}" local project="${1}"
local mirror="${2}" local mirror="${2}"
local bucket="${3}" local bucket="${3}"
local customer="${4}"
local mirrorbucket="${bucket}-mirror" local mirrorbucket="${bucket}-mirror"
local mountpoint="/opt/backupmounts/${bucket}" local mountpoint="/opt/backupmounts/${bucket}"
ps aux | grep duplicity | grep "[^a-zA-Z]${bucket}" > /dev/null ps aux | grep duplicity | grep "[^a-zA-Z]${bucket}" > /dev/null
local oktorun=$? # 1 == this bucket has no other bakup process in progress local oktorun=$? # 1 == this bucket has no other bakup process in progress
mkdir -p ${mountpoint} mkdir -p ${mountpoint}
[ ${oktorun} -ne 0 ] && rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h [[ ${oktorun} -ne 0 ]] && rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h
rclone mkdir ${mirror}:${mirrorbucket} rclone mkdir ${mirror}:${mirrorbucket}
[ ${oktorun} -ne 0 ] && duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \ [[ ${oktorun} -ne 0 ]] && duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \
--no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} && \
[[ -n "${extra_backup_jobs[${customer}]}" ]] && [[ -f ${extra_backup_jobs[${customer} ]] && ${extra_backup_jobs[${customer}]}
umount ${mountpoint} umount ${mountpoint}
rmdir ${mountpoint} rmdir ${mountpoint}
# Clean up # Clean up
@ -57,6 +61,7 @@ function do_backup {
for entry in "${projects[@]}"; do for entry in "${projects[@]}"; do
project=$(echo ${entry} | awk '{print $1}') project=$(echo ${entry} | awk '{print $1}')
mirror=$(echo ${entry} | awk '{print $2}') mirror=$(echo ${entry} | awk '{print $2}')
customer=$(echo ${entry} | awk '{print $3}')
for bucket in $(rclone lsd ${project}:/ | awk '{print $5}'); do for bucket in $(rclone lsd ${project}:/ | awk '{print $5}'); do
maybesize=$(timeout 30s rclone size --json ${project}:${bucket}) maybesize=$(timeout 30s rclone size --json ${project}:${bucket})
if [[ ${?} -eq 124 ]]; then if [[ ${?} -eq 124 ]]; then
@ -66,9 +71,9 @@ for entry in "${projects[@]}"; do
fi fi
# If bucket is above 50 GB we fork # If bucket is above 50 GB we fork
if [[ ${size} -gt $((${fork_limit} * 1000000000)) ]]; then if [[ ${size} -gt $((${fork_limit} * 1000000000)) ]]; then
do_backup ${project} ${mirror} ${bucket} & do_backup ${project} ${mirror} ${bucket} ${customer} &
else else
do_backup ${project} ${mirror} ${bucket} do_backup ${project} ${mirror} ${bucket} ${customer}
fi fi
done done
done done