Merge branch 'main' of git+ssh://platform.sunet.se:22022/Drive/sunetdrive

manifests/script.pp

@@ -56,6 +56,17 @@ class sunetdrive::script (
   }
   $site_name  = $config['site_name']
   $user_bucket_name  = $config['user_bucket_name']
+  if $config['user_scans'] {
+    $config['user_scans'].each |$job| {
+      sunet::scriptherder::cronjob { $job['name']:
+        cmd           => "ssh -t -l script ${job['server']} /usr/bin/sudo /usr/local/bin/occ ${job['container']} files:scan ${job['user']}",
+        hour          => $job['hour'],
+        minute        => $job['minute'],
+        ok_criteria   => ['exit_status=0','max_age=1d'],
+        warn_criteria => ['exit_status=1','max_age=2d'],
+      }
+    }
+  }
 
   # It is a start that will get us user buckets and primary buckets
   $backup_projects = $location
@@ -97,6 +108,10 @@ class sunetdrive::script (
     unless  => "python3 -m pip list | grep drive-utils | grep ${drive_version}",
     require => Package['python3'],
   }
+  file { '/opt/backups/scripts':
+    ensure  => directory,
+    mode => '0700'
+  }
   file { '/root/.ssh/':
     ensure => directory,
     mode   => '0700',
@@ -335,6 +350,13 @@ class sunetdrive::script (
       group   => 'root',
       mode    => '0700',
     }
+    file { '/opt/backups/scripts/hb.sh':
+      ensure  => file,
+      content => template('sunetdrive/script/backup-hb.erb.sh'),
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0700',
+    }
     sunet::scriptherder::cronjob { 'backupmultinodedb':
       cmd           => '/root/tasks/backupmultinodedb.sh',
       hour          => '2',

manifests/sitemonitornaemon.pp

@@ -45,6 +45,12 @@ class sunetdrive::sitemonitornaemon() {
     content => template('sunetdrive/monitor/sunetdrive_thruk_templates.conf.erb'),
     mode    => '0644',
   }
+  nagioscfg::service {'check_scriptherder':
+    hostgroup_name => ['nrpe'],
+    check_command  => 'check_nrpe_1arg_to30!check_scriptherder',
+    description    => 'Scriptherder Status',
+    contact_groups => ['naemon-admins'],
+  }
   nagioscfg::service {'check_galera_cluster':
     hostgroup_name => ['galera_monitor'],
     check_command  => 'check_nrpe_1arg!check_galera_cluster',

templates/application/config.php.erb

@@ -68,7 +68,7 @@ $CONFIG = array (
   'instanceid' => '<%= @instanceid %>',
   'integrity.check.disabled' => true,
   'log_type' => 'file',
-  'loglevel' => 0,
+  'loglevel' => 1,
   'lookup_server' => '<%= @lookup_server %>',
   'mail_domain' => '<%= @mail_domain %>',
   'mail_from_address' => '<%= @mail_from_address %>',

templates/lookup/docker-compose_lookup.yml.erb

@@ -13,7 +13,5 @@ services:
       - 89.46.20.75
       - 89.46.21.29
       - 89.32.32.32
-    ports:
-      - 443:443
     command: apachectl -D FOREGROUND
     tty: true

templates/script/backup-all-buckets.erb.sh

@@ -6,7 +6,9 @@ sleep $((16#$(ip a | grep "link/ether" | head -1 | awk -F ':' '{print $6}' | awk
 number_of_full_to_keep='<%= @full_backup_retention %>'
 fork_limit=30 #in GB, if bigger than this number, we fork the backup to it's own process
 customer="<%= @customer %>"
+environment="<%= @environment %>"
 declare -A extra_backup_jobs
+
 #<% if @extra_backup_jobs.any? %>
 #<% @extra_backup_jobs.each do |client, job| %>
 extra_backup_jobs["<%= client %>"]="<%= job %>"
@@ -25,9 +27,9 @@ projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= singl
 #<% end %>
 #<% end %>
 #<% else %>
-declare -a projects=("<%= @primary_project %> <%= @mirror_project %> <%= customer %>")
+declare -a projects=("<%= @primary_project %> <%= @mirror_project %> ${customer}")
 #<% @assigned_projects.each do |project| %>
-projects+=("<%= project['project'] %> <%= project['mirror_project'] %> <%= customer %>")
+projects+=("<%= project['project'] %> <%= project['mirror_project'] %> ${customer}")
 #<% end %>
 #<% end %>
 
@@ -45,17 +47,19 @@ function do_backup {
   local mountpoint="/opt/backupmounts/${bucket}"
   ps aux | grep duplicity | grep "[^a-zA-Z]${bucket}" > /dev/null
   local oktorun=$?   # 1 == this bucket has no other bakup process in progress
+  if [[ ${oktorun} -ne 0 ]]; then
     mkdir -p ${mountpoint}
-	[[ ${oktorun} -ne 0 ]] && rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h
+    rclone mount ${project}:${bucket} ${mountpoint}/ --daemon --allow-other --dir-cache-time 24h
     rclone mkdir ${mirror}:${mirrorbucket}
-	[[ ${oktorun} -ne 0 ]] && duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt \
-		--no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket} && \
-    [[ -n "${extra_backup_jobs[${customer}]}" ]] && [[ -f ${extra_backup_jobs[${customer} ]] && ${extra_backup_jobs[${customer}]}
+    duplicity --full-if-older-than 1M --asynchronous-upload --tempdir /mnt --archive-dir /mnt --no-encryption ${mountpoint} rclone://${mirror}:/${mirrorbucket}
+    if [[ "${extra_backup_jobs[${customer}]:+found}" == "found" ]]  && [[ -f "${extra_backup_jobs[${customer}]}" ]]; then
+      ${extra_backup_jobs[${customer}]} ${project} ${mirror} ${bucket} ${customer} ${environment}
+    fi
     umount ${mountpoint}
     rmdir ${mountpoint}
     # Clean up
-	[ ${oktorun} -ne 0 ] && duplicity remove-all-but-n-full ${number_of_full_to_keep} --tempdir /mnt --archive-dir /mnt \
-		--force rclone://${mirror}:/${mirrorbucket}
+    duplicity remove-all-but-n-full ${number_of_full_to_keep} --tempdir /mnt --archive-dir /mnt --force rclone://${mirror}:/${mirrorbucket}
+  fi
 }
 
 for entry in "${projects[@]}"; do

templates/script/backup-hb.erb.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+project="${1}"
+mirror="${2}"
+bucket="${3}"
+customer="${4}"
+environment="${5}"
+
+rsync -e "ssh -i ${HOME}/.ssh/id_script"  -avz "/opt/backupmounts/${bucket}"  "sd-${environment}@sd-${environment}-backup.hb.se:~/sd-${environment}/${bucket}"

templates/script/create_folders_in_fullnode_buckets.erb.sh

@@ -25,7 +25,7 @@ dirty=0
 primary=''
 declare -a users=( 'admin' )
 cd "${tempdir}" || echo "Could not cd to tempdir"
-declare -a projects=( "${yq}" -r '.project_mapping.'"${customer}"'.'"${environment}"'.assigned | "\(.[].project)"' /etc/hiera/data/common.yaml )
+declare -a projects=( $("${yq}" -r '.project_mapping.'"${customer}"'.'"${environment}"'.assigned | "\(.[].project)"' /etc/hiera/data/common.yaml) )
 if [[ "${include_userbuckets}" == "true" ]]; then
   primary=$("${yq}" -r '.project_mapping.'"${customer}"'.'"${environment}"'.primary_project' /etc/hiera/data/common.yaml)
   projects+=( "${primary}" )

templates/scriptreceiver/bashrc.erb.sh

@@ -83,7 +83,11 @@ alias la='ls -A'
 alias l='ls -CF'
 
 if [[ $(hostname) =~ redis ]]; then
+  if [ -f /opt/redis/node-0/server.conf ]; then
     alias redis-cli="redis-cli -a $(grep requirepass /opt/redis/node-0/server.conf | awk '{print $2}' | sed 's/"//g') --no-auth-warning"
+  else
+    alias redis-cli="redis-cli -a $(grep requirepass /opt/redict/node-0/server.conf | awk '{print $2}' | sed 's/"//g') --no-auth-warning"
+  fi
 fi
 
 # Alias definitions.