From 79baa3b4df969b14c3c16fdc57621422b554bd93 Mon Sep 17 00:00:00 2001 From: Rasmus Thorslund Date: Wed, 23 Apr 2025 10:38:29 +0200 Subject: [PATCH] added snapshot monitoring --- global/overlay/etc/puppet/cosmos-rules.yaml | 1 + .../modules/rut/manifests/monitor_nrpe.pp | 10 ++++++++ .../puppet/modules/rut/manifests/rut_mon.pp | 7 ++++++ .../rut/templates/check_rut_snapshots.sh.erb | 24 +++++++++++++++++++ .../overlay/etc/hiera/data/local.eyaml | 2 ++ 5 files changed, 44 insertions(+) create mode 100644 global/overlay/etc/puppet/modules/rut/manifests/monitor_nrpe.pp create mode 100644 global/overlay/etc/puppet/modules/rut/templates/check_rut_snapshots.sh.erb diff --git a/global/overlay/etc/puppet/cosmos-rules.yaml b/global/overlay/etc/puppet/cosmos-rules.yaml index f398a5e..78ff174 100644 --- a/global/overlay/etc/puppet/cosmos-rules.yaml +++ b/global/overlay/etc/puppet/cosmos-rules.yaml @@ -48,6 +48,7 @@ '^internal-sto4-prod-monitor-[0-9].rut.sunet.se$': sunet::dockerhost2: rut::rut_mon: + rut::monitor_nrpe: sunet::naemon_monitor: domain: monitor-prod.rut.sunet.se naemon_tag: latest diff --git a/global/overlay/etc/puppet/modules/rut/manifests/monitor_nrpe.pp b/global/overlay/etc/puppet/modules/rut/manifests/monitor_nrpe.pp new file mode 100644 index 0000000..d685000 --- /dev/null +++ b/global/overlay/etc/puppet/modules/rut/manifests/monitor_nrpe.pp @@ -0,0 +1,10 @@ +class rut::monitor_nrpe { + sunet::nagios::nrpe_command {'check_rut_snapshots': + command_line => '/usr/lib/nagios/plugins/check_rut_snapshots.sh' + } + file { "/usr/lib/nagios/plugins/check_rut_snapshots.sh": + ensure => "file", + content => template("rut/check_rut_snapshots.sh.erb"), + mode => '0755', + } +} diff --git a/global/overlay/etc/puppet/modules/rut/manifests/rut_mon.pp b/global/overlay/etc/puppet/modules/rut/manifests/rut_mon.pp index 6dc136c..9ef0607 100644 --- a/global/overlay/etc/puppet/modules/rut/manifests/rut_mon.pp +++ b/global/overlay/etc/puppet/modules/rut/manifests/rut_mon.pp @@ -4,9 +4,16 @@ class rut::rut_mon { check_command => 'check_nrpe!check_rut_pods', description => 'Microk8s cluster health', } + nagioscfg::service {'check_rut_postgres': host_name => ['internal-sto4-prod-k8sc-0.rut.sunet.se', 'internal-sto4-prod-k8sc-1.rut.sunet.se', 'internal-sto4-prod-k8sc-2.rut.sunet.se'], check_command => 'check_nrpe!check_rut_postgres', description => 'Postgres cluster health', } + + nagioscfg::service {'check_rut_snapshots': + host_name => ['internal-sto4-prod-monitor-1.rut.sunet.se'], + check_command => 'check_nrpe!check_rut_postgres', + description => 'Postgres cluster health', + } } diff --git a/global/overlay/etc/puppet/modules/rut/templates/check_rut_snapshots.sh.erb b/global/overlay/etc/puppet/modules/rut/templates/check_rut_snapshots.sh.erb new file mode 100644 index 0000000..9866922 --- /dev/null +++ b/global/overlay/etc/puppet/modules/rut/templates/check_rut_snapshots.sh.erb @@ -0,0 +1,24 @@ +#!/bin/bash +# file managed by puppet +# rut-mon-snapshots (on thorslunds SS account) + +export OS_AUTH_URL="https://v2.api.sto4.safedc.net:5000/v3/" +export OS_AUTH_TYPE="v3applicationcredential" +export OS_APPLICATION_CREDENTIAL_ID="<%= scope.call_function('safe_hiera',['sto4_application_credential_id']) %>" +export OS_APPLICATION_CREDENTIAL_SECRET="<%= scope.call_function('safe_hiera',['sto4_application_credential_secret']) %>" + + +export OS_REGION_NAME="sto4" +export OS_INTERFACE="public" + +num_snaps=$(openstack volume snapshot list -f json \ + | jq -r '.[].ID' \ + | wc -l | tr -d ' ') + +if [ "$num_snaps" -eq 0 ]; then + echo "OK: No Cinder snapshots found." + exit 0 +fi + +echo "CRITICAL: $num_snaps snapshot(s) present." +exit 2 diff --git a/internal-sto4-prod-monitor-1.rut.sunet.se/overlay/etc/hiera/data/local.eyaml b/internal-sto4-prod-monitor-1.rut.sunet.se/overlay/etc/hiera/data/local.eyaml index 4a63d1a..936905b 100644 --- a/internal-sto4-prod-monitor-1.rut.sunet.se/overlay/etc/hiera/data/local.eyaml +++ b/internal-sto4-prod-monitor-1.rut.sunet.se/overlay/etc/hiera/data/local.eyaml @@ -28,3 +28,5 @@ monitor-prod_rut_sunet_se_ssh_key: > 011e1IRQ3av9Cgtdi52Sqj7PBZ+IMIvnNs0937m9aX1v/gjDKXLcUFDV09I6 hUKE13Oa6nmBMJ7guBj6Obh+A9DLIv70LHnXvp5yGmtys1mZ3V1j1z5sebKW cfCoZr+p4zRTD6r47Xu2ZyyiTrrSw6FupMHw==] +sto4_application_credential_id: ENC[PKCS7,MIIDDQYJKoZIhvcNAQcDoIIC/jCCAvoCAQAxggKVMIICkQIBADB5MGExCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMjAwBgNVBAMMKWludGVybmFsLXN0bzQtcHJvZC1tb25pdG9yLTEucnV0LnN1bmV0LnNlAhRX/Ef9eDvc3IGBl6WsU5Sc4SdLUzANBgkqhkiG9w0BAQEFAASCAgAFhvP3soILEIz8XnP2mSNg0fBwZ/NsUyFJpU0mKFSVk5+JSyF0qbcul/E/D6Jyx00fTBlLSjLhah5XEffI9fJvEMTeSuN+bBbin830Tf1mrMlJF9rdSKFmbN7fJkMQ9MbK1uwxafDK+TstoGcipqzv271XIpsX0EShf6fFWx4KlZvhUU3p9EZyPStIB0Kqe8EmjrWIN6iEdhaNvSVKc1pbaRfNSbJc3+Ip0nykj4cWYD8S2gfL44ru1e/PTCffMQLf1ur/cG0hAU4u0W/DeOdBo0oyc97QcG4nwgfK1Mq7x0o0XeNc7g9ZIq2VIj9qX87T+PuUnOEW0kq5SBwDJS1ExO8aS3z5XykffZ5AS8OuhAV7fLLKEMprC/I4rrpU/GBizsN4GS9+E8VOEyZRnlz9wiaZ3k3argKQfhq9u9LehCNhq0IfPC0R9auGDhKnARYHGyGbqvJkuw92eYNRMwOeziWaAXNKIXbfE9W6vyewSX4IYdxsRXaN0tdTS77Lc5ArOgOYbaRP0/JDWNgNrkKXXEkWP08Cy40pyqy0bCQOPZIdnQwDlgHEdofI1OobPo70bHM2icpuGKw0abwUlb5DAqYtgp3OtlGiklJN1sBLtHAu5uUN6w2R5CVPaurfF89RcP9AErji1dOubV2hPTBq5YQiYrJtdwNeil/n/dnOCzBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBXSBKL01K6Z/+9BYcGZKmMgDDGvW2K5ZPc6K32060Xc+UHUjmJbKYpwZZvlCSZ0GYhmUlY3ykF5EYnG1BGMaY5aX0=] +sto4_application_credential_secret: ENC[PKCS7,MIIDPgYJKoZIhvcNAQcDoIIDLzCCAysCAQAxggKVMIICkQIBADB5MGExCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMjAwBgNVBAMMKWludGVybmFsLXN0bzQtcHJvZC1tb25pdG9yLTEucnV0LnN1bmV0LnNlAhRX/Ef9eDvc3IGBl6WsU5Sc4SdLUzANBgkqhkiG9w0BAQEFAASCAgCdaqIgqwWIOrrWdp7NWWIRIejEv4SsetlTAPKQ0RztzFqBF73uYnBxFl8rG8GGbF5h0z4eFTJhNrgsl+NNOaEq/E6Xq8QDma47QjdG/OEM83Nn+ULjoLzstuYCFRMuz9fYLzJJw2cUkOyg3N5gaI+imUdKm8OBH400V+C67VgnzYKgO0L2wAsaYrOiF4v1eU9hb4ZpSDwo9btmZNJILQ+nXP8P5qgS5O4ZIPaNp07KdpXtiyERfOTK+GbAVEbkURXnXiNs64Gq1DJBU5iueyZ8O9/RfKzuqGmfShjGQxob6DFBjHTnAJN2oYIespl0e9d/2lcCJ11s/e7Q+NC/D+n9HlvFWRowngAU/1sasUcAHNz0pRzgqK3n1f73ZrbeoRux4T3z4hMAOTucK0pdob3rrppEpbG1D941hxUjJj+NFYgBHJhXMuf6ln+Fwo/jfJhdPlhW+gX51nInK76FbrWN8wywLwCOLVFE5H7tVZSENq096atlp0O+R9Ocj/+X0sgsuRkuvw04Ekuyh5/XUELx9SvJ84lUlYHiSyCBxKvLFi29kt1qKE4QFIUboOcWDNEYOSZg2Ikxvr5j9fIj9yYfBN5cCBjrCeulXsQa1PnfnYP++GmbPS5FbwY0ZdNfDY79NWut8qavF5PfP2Bl7o73ZFDuCDk/WOSca6iQSgKh1TCBjAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQfjiDSvIn833zrmvc1IwRxoBgVt9VMzD6S+f8uZXOTwxfiRQ3xQW8bBl/Te5EjvYTAmMfQgHZBassBlMGFKau/fRv4tVewJ/QE0MloBNrNmGUzgqateWT5ACeCjUJpeC3Q+07X+ipxziIhQ0L6PNoaL54]