From 6cbfd488af6570d0173df8df798fb437d661ffd3 Mon Sep 17 00:00:00 2001
From: Maria Haider <mariah@sunet.se>
Date: Thu, 10 Nov 2022 11:59:43 +0100
Subject: [PATCH] add new eidastest check

Updated QA check
---
 .../etc/puppet/manifests/cosmos-site.pp       |  6 ++
 .../nagios/plugins/check_eidastest_prod.sh    | 83 +++++++++++++++++++
 .../lib/nagios/plugins/check_eidastest_qa.sh  | 33 +++++++-
 3 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 global/overlay/usr/lib/nagios/plugins/check_eidastest_prod.sh

diff --git a/global/overlay/etc/puppet/manifests/cosmos-site.pp b/global/overlay/etc/puppet/manifests/cosmos-site.pp
index 6bf70dcc..0ba01991 100644
--- a/global/overlay/etc/puppet/manifests/cosmos-site.pp
+++ b/global/overlay/etc/puppet/manifests/cosmos-site.pp
@@ -1366,6 +1366,12 @@ class nagios_monitor {
       description    => 'check test countires in QA',
       contact_groups => ['alerts'],
   }
+  nagioscfg::service {'check_eidastest_prod':
+      host_name      => ['localhost'],
+      check_command  => 'check_eidastest_prod',
+      description    => 'check test countires in Production',
+      contact_groups => ['alerts'],
+  }
   nagioscfg::command {'check_metadata_age':
       command_line   => "/usr/lib/nagios/plugins/check_eidas_metadata_age.sh '\$ARG1\$' '\$ARG2\$' '\$ARG3\$'"
   }
diff --git a/global/overlay/usr/lib/nagios/plugins/check_eidastest_prod.sh b/global/overlay/usr/lib/nagios/plugins/check_eidastest_prod.sh
new file mode 100644
index 00000000..3990f290
--- /dev/null
+++ b/global/overlay/usr/lib/nagios/plugins/check_eidastest_prod.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+
+exec_status=$(curl -s https://api.swedenconnect.se/testid/prod/execution-status | jq --raw-output)
+
+exec_status_result=$?
+
+
+if [[ $exec_status_result -ne 0 ]]; then
+   echo "CRITICAL - Service FAIL"
+   exit $STATE_CRITICAL
+fi
+
+calculate_time() {
+     md_date=$(date -d "$1" +%s)
+     now=$(date +%s)
+     diff=$(("$md_date"-"$now"))
+
+     if ((diff < 0)); then ((diff*=-1)); fi
+     echo $diff
+}
+
+exec_time=$(echo "$exec_status" | jq --raw-output '."last execution result".executiontimestamp')
+
+diff_exec_time=$(calculate_time "$exec_time")
+
+day=$(( 24 * 3600))
+
+if [ "$diff_exec_time" -gt "$day" ]; then
+   echo "CRITICAL - tests are too old!!"
+   exit $STATE_CRITICAL
+
+fi
+
+raw_lands=$(echo "$exec_status" | jq --raw-output '."last execution result".failed[].result')
+
+declare -a lands=($(echo "$raw_lands" | tr "\n" " "))
+
+exit_status_warning=0
+exit_status_critical=0
+output=""
+
+for land in "${lands[@]}"
+do
+ if [ "$land" ]
+ then
+     first_failed=$(curl -s https://api.swedenconnect.se/testid/prod/failed-tests | jq --raw-output '."failed tests"[].'\"$land\"'|select( . != null )')
+
+     diff_failed=$(calculate_time "$first_failed")
+
+     half_day=$((12 * 3600))
+     sec_to_hrs=$(("$diff_failed" / 3600))
+
+     if [ "$diff_failed" -gt "$day" ]; then
+        exit_status_critical=1
+        output="$output CRITICAL - $land is failing for $sec_to_hrs hours!"
+
+     elif [ "$diff_failed" -gt "$half_day" ]; then
+        exit_status_warning=1
+        output="$output WARNING - $land is failing for $sec_to_hrs hours!"
+     fi
+ fi
+done
+
+if [ "$exit_status_critical" -gt 0 ]
+then
+    echo "$output"
+    exit $STATE_CRITICAL
+
+elif [ "$exit_status_warning" -gt 0 ]
+then
+    echo "$output"
+    exit $STATE_WARNING
+
+elif [ -z "${lands[*]}" ]
+then
+   echo "Tests on production lands are error free"
+
+else
+   echo "${lands[*]} failing"
+fi
+
+exit 0
diff --git a/global/overlay/usr/lib/nagios/plugins/check_eidastest_qa.sh b/global/overlay/usr/lib/nagios/plugins/check_eidastest_qa.sh
index fe2dff1f..66d544e0 100755
--- a/global/overlay/usr/lib/nagios/plugins/check_eidastest_qa.sh
+++ b/global/overlay/usr/lib/nagios/plugins/check_eidastest_qa.sh
@@ -1,5 +1,36 @@
 #!/bin/bash
 
+exec_status=$(curl -s https://api.swedenconnect.se/testid/qa/execution-status | jq --raw-output)
+
+exec_status_result=$?
+
+
+if [[ $exec_status_result -ne 0 ]]; then
+   echo "CRITICAL - Service FAIL"
+   exit $STATE_CRITICAL
+fi
+
+calculate_time() {
+     md_date=$(date -d "$1" +%s)
+     now=$(date +%s)
+     diff=$(("$md_date"-"$now"))
+
+     if ((diff < 0)); then ((diff*=-1)); fi
+     echo $diff
+}
+
+exec_time=$(echo "$exec_status" | jq --raw-output '."last execution result".executiontimestamp')
+
+diff_exec_time=$(calculate_time "$exec_time")
+
+day=$(( 24 * 3600))
+
+if [ "$diff_exec_time" -gt "$day" ]; then
+   echo "CRITICAL - tests are too old!!"
+   exit $STATE_CRITICAL
+
+fi
+
 
 raw_lands=$(curl -s https://api.swedenconnect.se/testid/qa/execution-status | jq --raw-output '."last execution result".failed[].result')
 
@@ -24,7 +55,7 @@ done
 
 if [ "$exit_status" -gt 0 ]
 then
-    echo $output
+    echo "$output"
 else
    echo "Tests on test lands are error free"
 fi