From 5bc0ddfa00aa53d20ed0fb70d4f0f220a906310d Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 09:28:39 +0200 Subject: [PATCH 1/8] Add admin password as env variable to image --- templates/application/docker-compose_nextcloud.yml.erb | 2 ++ templates/multinode/docker-compose_nextcloud.yml.erb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/templates/application/docker-compose_nextcloud.yml.erb b/templates/application/docker-compose_nextcloud.yml.erb index 4773587..e6d4468 100644 --- a/templates/application/docker-compose_nextcloud.yml.erb +++ b/templates/application/docker-compose_nextcloud.yml.erb @@ -5,6 +5,8 @@ services: app: image: docker.sunet.se/drive/nextcloud-custom:<%= @nextcloud_version %> restart: always + environment: + - NC_PASS=<%= @admin_password%> volumes: - /opt/nextcloud/000-default.conf:/etc/apache2/sites-enabled/000-default.conf - /opt/nextcloud/mpm_prefork.conf:/etc/apache2/mods-available/mpm_prefork.conf diff --git a/templates/multinode/docker-compose_nextcloud.yml.erb b/templates/multinode/docker-compose_nextcloud.yml.erb index 69d29e7..c6f0bb3 100644 --- a/templates/multinode/docker-compose_nextcloud.yml.erb +++ b/templates/multinode/docker-compose_nextcloud.yml.erb @@ -14,6 +14,8 @@ services: - <%= @config_php_path %>:/var/www/html/config/config.php - <%= @nextcloud_log_path %>:/var/www/html/data/nextcloud.log - <%= @rclone_conf_path %>:/rclone.conf + environment: + - NC_PASS=<%= @admin_password%> networks: - default - proxysql_proxysql From ac6430df513364105c160a0a6760174038950c39 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 12:08:08 +0200 Subject: [PATCH 2/8] Add announce script to script servers --- manifests/script.pp | 22 +++++- templates/script/announce.erb.sh | 75 +++++++++++++++++++++ templates/script/multinodeannounce.erb.sh | 81 +++++++++++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100755 templates/script/announce.erb.sh create mode 100755 templates/script/multinodeannounce.erb.sh diff --git a/manifests/script.pp b/manifests/script.pp index ccb03ad..9f3b38a 100644 --- a/manifests/script.pp +++ b/manifests/script.pp @@ -18,7 +18,8 @@ class sunetdrive::script ( $backup_server = $config['backup_server'] $rclone_url = 'https://downloads.rclone.org/rclone-current-linux-amd64.deb' $local_path = '/tmp/rclone-current-linux-amd64.deb' - $singlenodes = hiera('singlenodes') + $singlenodes = lookup('singlenodes') + $multinodes = keys(lookup('multinode_mapping')) if $customer == 'mdu' { $eppn_suffix = 'mdh.se' @@ -297,6 +298,16 @@ class sunetdrive::script ( } } if $customer == 'common' { + $multinode_passwords = $multinodes.map | $index, $customer | { + safe_hiera("${customer}_admin_app_password") + } + file { '/root/tasks/announce.sh': + ensure => file, + content => template('sunetdrive/script/multinodeannounce.erb.sh'), + owner => 'root', + group => 'root', + mode => '0700', + } file { '/root/tasks/backupmultinodedb.sh': ensure => file, content => template('sunetdrive/script/backupmultinodedb.erb.sh'), @@ -326,7 +337,7 @@ class sunetdrive::script ( ok_criteria => ['exit_status=0','max_age=2d'], warn_criteria => ['exit_status=1','max_age=3d'], } - } + } $singlenodes.each | $singlenode| { $multinode = hiera_hash('multinode_mapping')[$singlenode]['server'] $multinodeserver = "${multinode}.${site_name}" @@ -377,6 +388,13 @@ class sunetdrive::script ( warn_criteria => ['exit_status=1','max_age=3d'], } } else { + file { '/root/tasks/announce.sh': + ensure => file, + content => template('sunetdrive/script/announce.erb.sh'), + owner => 'root', + group => 'root', + mode => '0700', + } file { '/root/tasks/backupmultinodedb.sh': ensure => absent, } diff --git a/templates/script/announce.erb.sh b/templates/script/announce.erb.sh new file mode 100755 index 0000000..d9ae874 --- /dev/null +++ b/templates/script/announce.erb.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +VALID_ARGS=$(getopt -o cdghi:m:s: --long create,delete,get,help,id:,message:,subject: -- "$@") +# shellcheck disable=SC2181 +if [[ ${?} -ne 0 ]]; then + exit 1; +fi + +usage () { + echo "${0}: -c|--create -m|--message <'Your announcement goes here'> -s|--subject " + echo "${0}: -d|--delete -i|--id " + echo "${0}: -g|--get" + exit 1 +} + +eval set -- "${VALID_ARGS}" +# shellcheck disable=SC2078 +while [ : ]; do + case "$1" in + -c | --create) + method='POST' + shift + ;; + -d | --delete) + method='DELETE' + shift + ;; + -g | --get) + method='GET' + shift + ;; + -h | --help) + usage + ;; + -i | --id) + argument="${2}" + shift 2 + ;; + -m | --message) + message="${2}" + shift 2 + ;; + -s | --subject) + subject="${2}" + shift 2 + ;; + *) + break + ;; + esac +done + +if [[ ${method} == 'DELETE' ]] && [[ -z ${argument} ]]; then + usage +fi +if [[ ${method} == 'POST' ]]; then + if [[ -z ${message} ]] || [[ -z ${subject} ]]; then + usage + fi + argument='{"subject":"'${subject}'","message":"'${message}'", "plainMessage":"'${message}'", "groups": [], "userId": "admin", "activities": false, "notifications": true, "emails": false, "comments": false }' +fi + +curl_cmd(){ + local method="${1}" + if [[ ${method} == 'POST' ]] && [[ -n ${2} ]]; then + local payload=(-d "${2}" -H "Content-Type: application/json") + elif [[ ${method} == 'DELETE' ]] && [[ -n ${2} ]]; then + local id="/${2}" + fi + local admin_app_password="<%= @admin_app_password %>" + domain="$(hostname -d)" + curl -X "${method}" -u "admin:${admin_app_password}" "${payload[@]}" -H 'OCS-APIRequest: true' "https://${domain}/ocs/v2.php/apps/announcementcenter/api/v1/announcements${id}" +} +curl_cmd "${method}" "${argument}" + diff --git a/templates/script/multinodeannounce.erb.sh b/templates/script/multinodeannounce.erb.sh new file mode 100755 index 0000000..a04d0fc --- /dev/null +++ b/templates/script/multinodeannounce.erb.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +VALID_ARGS=$(getopt -o cdghi:m:s: --long create,delete,get,help,id:,message:,subject: -- "$@") +# shellcheck disable=SC2181 +if [[ ${?} -ne 0 ]]; then + exit 1; +fi + +usage () { + echo "${0}: -c|--create -m|--message <'Your announcement goes here'> -s|--subject " + echo "${0}: -d|--delete -i|--id " + echo "${0}: -g|--get" + exit 1 +} + +eval set -- "${VALID_ARGS}" +# shellcheck disable=SC2078 +while [ : ]; do + case "$1" in + -c | --create) + method='POST' + shift + ;; + -d | --delete) + method='DELETE' + shift + ;; + -g | --get) + method='GET' + shift + ;; + -h | --help) + usage + ;; + -i | --id) + argument="${2}" + shift 2 + ;; + -m | --message) + message="${2}" + shift 2 + ;; + -s | --subject) + subject="${2}" + shift 2 + ;; + *) + break + ;; + esac +done + +if [[ ${method} == 'DELETE' ]] && [[ -z ${argument} ]]; then + usage +fi +if [[ ${method} == 'POST' ]]; then + if [[ -z ${message} ]] || [[ -z ${subject} ]]; then + usage + fi + argument='{"subject":"'${subject}'","message":"'${message}'", "plainMessage":"'${message}'", "groups": [], "userId": "admin", "activities": false, "notifications": true, "emails": false, "comments": false }' +fi + +curl_cmd(){ + local admin_app_password="${1}" + local customer="${2}" + local method="${3}" + if [[ ${method} == 'POST' ]] && [[ -n ${4} ]]; then + local payload=(-d "${4}" -H "Content-Type: application/json") + elif [[ ${method} == 'DELETE' ]] && [[ -n ${4} ]]; then + local id="/${4}" + fi + domain="$(hostname -d)" + curl -X "${method}" -u "admin:${admin_app_password}" "${payload[@]}" -H 'OCS-APIRequest: true' "https://${customer}.${domain}/ocs/v2.php/apps/announcementcenter/api/v1/announcements${id}" +} + +#<%- index = 0 -%> +#<%- @multinodes.each do |customer| -%> +curl_cmd "<%= @multinode_passwords[index] %>" "<%= customer %>" "${method}" "${argument}" +#<%- index += 1 -%> +#<%- end -%> + From b927716aadf83d4aa7ab86cb967bedd8320416e8 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 12:12:59 +0200 Subject: [PATCH 3/8] Fix comment --- templates/script/multinodeannounce.erb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/script/multinodeannounce.erb.sh b/templates/script/multinodeannounce.erb.sh index a04d0fc..15a1f8e 100755 --- a/templates/script/multinodeannounce.erb.sh +++ b/templates/script/multinodeannounce.erb.sh @@ -74,8 +74,8 @@ curl_cmd(){ } #<%- index = 0 -%> -#<%- @multinodes.each do |customer| -%> +#<%- @multinodes.each do |customer| %> Run <%= customer%> curl_cmd "<%= @multinode_passwords[index] %>" "<%= customer %>" "${method}" "${argument}" #<%- index += 1 -%> -#<%- end -%> +#<%- end %> End of <%= customer%> From d540c90afa2939c7d8ecd6326362098d14c2a13e Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 12:15:54 +0200 Subject: [PATCH 4/8] Try again --- templates/script/multinodeannounce.erb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/script/multinodeannounce.erb.sh b/templates/script/multinodeannounce.erb.sh index 15a1f8e..b85c4d5 100755 --- a/templates/script/multinodeannounce.erb.sh +++ b/templates/script/multinodeannounce.erb.sh @@ -74,8 +74,8 @@ curl_cmd(){ } #<%- index = 0 -%> -#<%- @multinodes.each do |customer| %> Run <%= customer%> +#<%- @multinodes.each do |customer| %> curl_cmd "<%= @multinode_passwords[index] %>" "<%= customer %>" "${method}" "${argument}" #<%- index += 1 -%> -#<%- end %> End of <%= customer%> +#<%- end %> From c7bd0fa9657b8978ab78fba5508c6eb74c17a00b Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 12:19:33 +0200 Subject: [PATCH 5/8] Try again --- templates/script/multinodeannounce.erb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/script/multinodeannounce.erb.sh b/templates/script/multinodeannounce.erb.sh index b85c4d5..a11b1ef 100755 --- a/templates/script/multinodeannounce.erb.sh +++ b/templates/script/multinodeannounce.erb.sh @@ -73,9 +73,9 @@ curl_cmd(){ curl -X "${method}" -u "admin:${admin_app_password}" "${payload[@]}" -H 'OCS-APIRequest: true' "https://${customer}.${domain}/ocs/v2.php/apps/announcementcenter/api/v1/announcements${id}" } -#<%- index = 0 -%> +#<%- index = 0 %> #<%- @multinodes.each do |customer| %> curl_cmd "<%= @multinode_passwords[index] %>" "<%= customer %>" "${method}" "${argument}" -#<%- index += 1 -%> +#<%- index += 1 %> #<%- end %> From 7207ef2bbff13b47d63d44061d86dabc5d61c9e2 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 12:30:16 +0200 Subject: [PATCH 6/8] Allways backup the db --- templates/script/restart-nextcloud-farm.erb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/templates/script/restart-nextcloud-farm.erb b/templates/script/restart-nextcloud-farm.erb index f7832ce..d68c8a4 100755 --- a/templates/script/restart-nextcloud-farm.erb +++ b/templates/script/restart-nextcloud-farm.erb @@ -68,6 +68,7 @@ def main() -> int: apikey_test = "<%= @apikey_test %>" apikey_prod = "<%= @apikey_prod %>" + backup_command = ['sudo /home/script/bin/backup_db.sh'] cosmos_command = ['sudo run-cosmos'] nc_upgrade_command = 'sudo /usr/local/bin/occ config:editable --on ' nc_upgrade_command += '&& sudo /usr/local/bin/occ upgrade ' @@ -80,9 +81,18 @@ def main() -> int: repair_command += '&& sudo /usr/local/bin/occ db:add-missing-primary-keys' reboot_command = ['sudo /usr/local/bin/safer_reboot'] server_type = "node" + backup_type = "backup" if customer == "common": customer = "gss" server_type = "gss" + backup_type = "gssbackup" + + backup = build_fqdn(customer, environment, 1, backup_type) + print("\tRunning backup command at {}".format(backup)) + run_remote_command(backup, + backup_command, + user="script", + output=subprocess.DEVNULL) for number in reversed(range(1, 4)): fqdn = build_fqdn(customer, environment, number, server_type) From 1b60d41dab8c2dfb492f9a1a62f2e01b379ea0c4 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 12:38:34 +0200 Subject: [PATCH 7/8] Get password --- manifests/script.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/script.pp b/manifests/script.pp index 9f3b38a..fb1d536 100644 --- a/manifests/script.pp +++ b/manifests/script.pp @@ -388,6 +388,7 @@ class sunetdrive::script ( warn_criteria => ['exit_status=1','max_age=3d'], } } else { + $admin_app_password = safe_hiera('admin_app_password') file { '/root/tasks/announce.sh': ensure => file, content => template('sunetdrive/script/announce.erb.sh'), From 22f5dfac6242981c97f73cabd013e346b25ac4a8 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Sat, 2 Sep 2023 12:50:08 +0200 Subject: [PATCH 8/8] Allways backup before restarting --- templates/script/restart-db-cluster.erb | 39 +++++++++++++++++++------ 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/templates/script/restart-db-cluster.erb b/templates/script/restart-db-cluster.erb index af552cc..c5ab40a 100755 --- a/templates/script/restart-db-cluster.erb +++ b/templates/script/restart-db-cluster.erb @@ -1,12 +1,12 @@ #!/usr/bin/env python3.9 # vim: set filetype=python: +import subprocess import sys import time import urllib.parse import requests - from drive_utils import (build_fqdn, get_ips_for_hostname, run_remote_command, smoketest_db_node) @@ -57,39 +57,60 @@ def main() -> int: apikey_prod = "<%= @apikey_prod %>" user = "script" + backup_command = ['sudo /home/script/bin/backup_db.sh'] reboot_command = ['sudo /usr/local/bin/safer_reboot'] if customers[0] == "common": customers = ["gss", "lookup", "multinode"] for customer in customers: + backup_type = "backup" + if customer == "gss": + backup_type = "gssbackup" + elif customer == "lookup": + backup_type = "lookupbackup" + elif customer == "multinode": + backup_command = ['sudo /home/script/bin/backup_multinode_db.sh'] + backup_type = "multinode-db" + + backup = build_fqdn(customer, environment, 1, backup_type) + print("\tRunning backup command at {}".format(backup)) + run_remote_command(backup, + backup_command, + user="script", + output=subprocess.DEVNULL) for number in reversed(range(1, 4)): fqdn = build_fqdn(customer, environment, number) ipv4, _ = get_ips_for_hostname(fqdn) ip = ipv4[0] print("Upgrading: {} with ip: {}".format(fqdn, ip)) add_downtime(fqdn, apikey_test) - add_downtime(fqdn, apikey_prod, monitor_host="monitor.drive.sunet.se") + add_downtime(fqdn, + apikey_prod, + monitor_host="monitor.drive.sunet.se") - run_remote_command(fqdn, reboot_command, user = user) + run_remote_command(fqdn, reboot_command, user=user) success = False for testnumber in reversed(range(1, 32, 2)): - print("\tSleeping for {} seconds before smoketest on {}".format( - testnumber, fqdn)) + print( + "\tSleeping for {} seconds before smoketest on {}".format( + testnumber, fqdn)) time.sleep(testnumber) - if smoketest_db_node(fqdn, user = user): + if smoketest_db_node(fqdn, user=user): success = True break remove_downtime(fqdn, apikey_test) - remove_downtime(fqdn, apikey_prod, monitor_host="monitor.drive.sunet.se") + remove_downtime(fqdn, + apikey_prod, + monitor_host="monitor.drive.sunet.se") if success: print("Upgrade cycle succeeded on {} ".format(fqdn)) else: - print("Smoketest failed on {} after server reboot command".format( - fqdn)) + print("Smoketest failed on {} after server reboot command". + format(fqdn)) return 5 print("All {}-servers successfully upgraded for {}".format(