Drive/sunetdrive
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Allways backup before restarting

Browse source
This commit is contained in:
Micke Nordin 2023-09-02 12:50:08 +02:00
parent 1b60d41dab
commit 22f5dfac62
Signed by untrusted user: Micke
GPG key ID: 0DA0A7A5708FE257
1 changed files with 30 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
templates/script/restart-db-cluster.erb
View file

@ -1,12 +1,12 @@
#!/usr/bin/env python3.9 #!/usr/bin/env python3.9
# vim: set filetype=python: # vim: set filetype=python:
import subprocess
import sys import sys
import time import time
import urllib.parse import urllib.parse
import requests import requests
from drive_utils import (build_fqdn, get_ips_for_hostname, run_remote_command, from drive_utils import (build_fqdn, get_ips_for_hostname, run_remote_command,
smoketest_db_node) smoketest_db_node)
@ -57,39 +57,60 @@ def main() -> int:
apikey_prod = "<%= @apikey_prod %>" apikey_prod = "<%= @apikey_prod %>"
user = "script" user = "script"
backup_command = ['sudo /home/script/bin/backup_db.sh']
reboot_command = ['sudo /usr/local/bin/safer_reboot'] reboot_command = ['sudo /usr/local/bin/safer_reboot']
if customers[0] == "common": if customers[0] == "common":
customers = ["gss", "lookup", "multinode"] customers = ["gss", "lookup", "multinode"]
for customer in customers: for customer in customers:
backup_type = "backup"
if customer == "gss":
backup_type = "gssbackup"
elif customer == "lookup":
backup_type = "lookupbackup"
elif customer == "multinode":
backup_command = ['sudo /home/script/bin/backup_multinode_db.sh']
backup_type = "multinode-db"
backup = build_fqdn(customer, environment, 1, backup_type)
print("\tRunning backup command at {}".format(backup))
run_remote_command(backup,
backup_command,
user="script",
output=subprocess.DEVNULL)
for number in reversed(range(1, 4)): for number in reversed(range(1, 4)):
fqdn = build_fqdn(customer, environment, number) fqdn = build_fqdn(customer, environment, number)
ipv4, _ = get_ips_for_hostname(fqdn) ipv4, _ = get_ips_for_hostname(fqdn)
ip = ipv4[0] ip = ipv4[0]
print("Upgrading: {} with ip: {}".format(fqdn, ip)) print("Upgrading: {} with ip: {}".format(fqdn, ip))
add_downtime(fqdn, apikey_test) add_downtime(fqdn, apikey_test)
add_downtime(fqdn, apikey_prod, monitor_host="monitor.drive.sunet.se") add_downtime(fqdn,
apikey_prod,
monitor_host="monitor.drive.sunet.se")
run_remote_command(fqdn, reboot_command, user = user) run_remote_command(fqdn, reboot_command, user=user)
success = False success = False
for testnumber in reversed(range(1, 32, 2)): for testnumber in reversed(range(1, 32, 2)):
print("\tSleeping for {} seconds before smoketest on {}".format( print(
testnumber, fqdn)) "\tSleeping for {} seconds before smoketest on {}".format(
testnumber, fqdn))
time.sleep(testnumber) time.sleep(testnumber)
if smoketest_db_node(fqdn, user = user): if smoketest_db_node(fqdn, user=user):
success = True success = True
break break
remove_downtime(fqdn, apikey_test) remove_downtime(fqdn, apikey_test)
remove_downtime(fqdn, apikey_prod, monitor_host="monitor.drive.sunet.se") remove_downtime(fqdn,
apikey_prod,
monitor_host="monitor.drive.sunet.se")
if success: if success:
print("Upgrade cycle succeeded on {} ".format(fqdn)) print("Upgrade cycle succeeded on {} ".format(fqdn))
else: else:
print("Smoketest failed on {} after server reboot command".format( print("Smoketest failed on {} after server reboot command".
fqdn)) format(fqdn))
return 5 return 5
print("All {}-servers successfully upgraded for {}".format( print("All {}-servers successfully upgraded for {}".format(