Update restore related scripts

global/overlay/etc/puppet/modules/net/files/baas2_restoretest/run-failed-restore.expect

@@ -15,12 +15,17 @@ expect "Action \\\[1,2,3,A\\\] : " {
   expect -exact "Confirm encryption key password: "
   sleep 1
   send -- "$enc_password\r"
-}
-expect "ANS8013I Invalid encryption key password" {
-  set exit_val 0
-  expect "Action \\\[1,2,3,A\\\] : "
-  send -- "A\r"
+
+  expect {
+    "ANS8013I Invalid encryption key password" {
+       set exit_val 0
+       expect "Action \\\[1,2,3,A\\\] : "
+       send -- "A\r"
+    }
+    eof {
+      exit $exit_val
+    }
+  }
 }
 
-expect eof
 exit $exit_val

global/overlay/etc/puppet/modules/net/files/baas2_restoretest/run-successful-restore.expect

@@ -3,7 +3,6 @@ set nodename [lindex $argv 0]
 set enc_password [lindex $argv 1]
 
 set exit_val 1
-
 spawn dsmc restore /opt/backup-test/ /opt/restore-target/ -asnodename=$nodename -subdir=yes
 expect "Action \\\[1,2,3,A\\\] : " {
     send -- "1\r"
@@ -15,9 +14,16 @@ expect "Action \\\[1,2,3,A\\\] : " {
     expect -exact "Confirm encryption key password: "
     sleep 1
     send -- "$enc_password\r"
-}
-expect "Restore processing finished." {
- set exit_val 0
+
+    expect {
+        "Restore processing finished." {
+            set exit_val 0
+        }
+        "Action \\\[1,2,3,A\\\] : " {
+            send -- "A\r"
+            expect eof
+        }
+    }
 }
 
 exit $exit_val

global/overlay/etc/puppet/modules/net/files/baas2_restoretest/validate-backup-files

@@ -2,6 +2,7 @@
 # This script is intended to validate that:
 #  - restore of an encrypted backup is unsuccessuful if wrong encrypted password is provided
 #  - restore of an encypted backup is successfil if the correct encryption password is provided
+#  - validate checksums generated by the backup node after the files are restored
 nodename=$1
 
 if [ -z "$nodename" ]; then
@@ -9,7 +10,7 @@ if [ -z "$nodename" ]; then
   exit 1
 fi
 
-set -eu
+set -u
 
 restoretarget=/opt/restore-target
 status_file=/opt/baas2/validate-backup-status
@@ -18,38 +19,44 @@ status_critical="critical"
 status_ok="ok"
 
 function exit_if_failed() {
-  if [ $1 -ne 0 ]; then
+  if [ "$1" -ne 0 ]; then
+    echo "writing status '$status_critical' to $status_file"
     echo $status_critical > "$status_file.tmp"
-    mv $status_file.tmp $status_file
+    mv -v $status_file.tmp $status_file
     exit 1
   fi
 }
 
 # We want to start clean without files for each run to avoid getting questions about replace
 rm -f $restoretarget/*
-exit_if_failed "$?"
+exit_if_failed $?
 
 # Get the encryption password for the server that we want to do "proxy" restore from
 enc_password=$(eyaml decrypt -f /etc/hiera/data/local.eyaml --pkcs7-private-key=/etc/hiera/eyaml/private_key.pkcs7.pem --pkcs7-public-key=/etc/hiera/eyaml/public_certkey.pkcs7.pem | awk '/^baas2_restoretest_encryption_password: /{print $2}')
 
+# Run expect script that cancel any pending/open restores (we can not be sure of the state, a previous failure can result in a pending restore)
+/opt/baas2/cancel-restore.expect "$nodename"
+exit_if_failed $?
+
 # Run expect script that should do a failed restore (wrong password) (and exit 0 if it failes)
 /opt/baas2/run-failed-restore.expect "$nodename"
-exit_if_failed "$?"
+exit_if_failed $?
 
 # Run expect script that cancel any pending/open restores (which is an excpected result of the failed restore above)
 /opt/baas2/cancel-restore.expect "$nodename"
-exit_if_failed "$?"
+exit_if_failed $?
 
 # Run expect script that uses the correct encryption password and does a successful restore
 /opt/baas2/run-successful-restore.expect "$nodename" "$enc_password"
-exit_if_failed "$?"
+exit_if_failed $?
 
 # Validate checksums genereted by the backup node for all restored files
-cd $restoretarget
+cd $restoretarget || exit_if_failed $?
 sha256sum -c $checksum_file
-exit_if_failed "$?"
+exit_if_failed $?
 
 # If all is good, exit 0
+echo "writing status '$status_ok' to $status_file"
 echo $status_ok > "$status_file.tmp"
 mv $status_file.tmp $status_file
-exit 0
+exit 0