From 22f5dfac6242981c97f73cabd013e346b25ac4a8 Mon Sep 17 00:00:00 2001
From: Micke Nordin <kano@sunet.se>
Date: Sat, 2 Sep 2023 12:50:08 +0200
Subject: [PATCH] Allways backup before restarting

---
 templates/script/restart-db-cluster.erb | 39 +++++++++++++++++++------
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/templates/script/restart-db-cluster.erb b/templates/script/restart-db-cluster.erb
index af552cc..c5ab40a 100755
--- a/templates/script/restart-db-cluster.erb
+++ b/templates/script/restart-db-cluster.erb
@@ -1,12 +1,12 @@
 #!/usr/bin/env python3.9
 # vim: set filetype=python:
 
+import subprocess
 import sys
 import time
 import urllib.parse
 
 import requests
-
 from drive_utils import (build_fqdn, get_ips_for_hostname, run_remote_command,
                          smoketest_db_node)
 
@@ -57,39 +57,60 @@ def main() -> int:
     apikey_prod = "<%= @apikey_prod %>"
     user = "script"
 
+    backup_command = ['sudo /home/script/bin/backup_db.sh']
     reboot_command = ['sudo /usr/local/bin/safer_reboot']
 
     if customers[0] == "common":
         customers = ["gss", "lookup", "multinode"]
     for customer in customers:
+        backup_type = "backup"
+        if customer == "gss":
+            backup_type = "gssbackup"
+        elif customer == "lookup":
+            backup_type = "lookupbackup"
+        elif customer == "multinode":
+            backup_command = ['sudo /home/script/bin/backup_multinode_db.sh']
+            backup_type = "multinode-db"
+
+        backup = build_fqdn(customer, environment, 1, backup_type)
+        print("\tRunning backup command at {}".format(backup))
+        run_remote_command(backup,
+                           backup_command,
+                           user="script",
+                           output=subprocess.DEVNULL)
         for number in reversed(range(1, 4)):
             fqdn = build_fqdn(customer, environment, number)
             ipv4, _ = get_ips_for_hostname(fqdn)
             ip = ipv4[0]
             print("Upgrading: {} with ip: {}".format(fqdn, ip))
             add_downtime(fqdn, apikey_test)
-            add_downtime(fqdn, apikey_prod, monitor_host="monitor.drive.sunet.se")
+            add_downtime(fqdn,
+                         apikey_prod,
+                         monitor_host="monitor.drive.sunet.se")
 
-            run_remote_command(fqdn, reboot_command, user = user)
+            run_remote_command(fqdn, reboot_command, user=user)
             success = False
 
             for testnumber in reversed(range(1, 32, 2)):
-                print("\tSleeping for {} seconds before smoketest on {}".format(
-                    testnumber, fqdn))
+                print(
+                    "\tSleeping for {} seconds before smoketest on {}".format(
+                        testnumber, fqdn))
                 time.sleep(testnumber)
 
-                if smoketest_db_node(fqdn, user = user):
+                if smoketest_db_node(fqdn, user=user):
                     success = True
 
                     break
 
             remove_downtime(fqdn, apikey_test)
-            remove_downtime(fqdn, apikey_prod, monitor_host="monitor.drive.sunet.se")
+            remove_downtime(fqdn,
+                            apikey_prod,
+                            monitor_host="monitor.drive.sunet.se")
             if success:
                 print("Upgrade cycle succeeded on {} ".format(fqdn))
             else:
-                print("Smoketest failed on {} after server reboot command".format(
-                    fqdn))
+                print("Smoketest failed on {} after server reboot command".
+                      format(fqdn))
 
                 return 5
         print("All {}-servers successfully upgraded for {}".format(