From 3fbfe26c661b7ee373ef35fbd2d3e05f82db93c4 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Wed, 5 Mar 2025 12:49:32 +0100
Subject: [PATCH 01/12] adding check for nextcloud mounts

---
 manifests/app_type.pp                         | 19 ++++++++++-
 manifests/sitemonitornaemon.pp                |  7 +++-
 .../application/check_nextcloud_mounts.py     | 34 +++++++++++++++++++
 3 files changed, 58 insertions(+), 2 deletions(-)
 create mode 100644 templates/application/check_nextcloud_mounts.py

diff --git a/manifests/app_type.pp b/manifests/app_type.pp
index 1996271..521bfbf 100644
--- a/manifests/app_type.pp
+++ b/manifests/app_type.pp
@@ -388,5 +388,22 @@ define sunetdrive::app_type (
       }
     }
   }
-
+  if $facts['networking']['hostname'] == 'node1' or $is_multinode {
+    file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py':
+      ensure  => present,
+      force   => true,
+      owner   => 'root',
+      group   => 'root',
+      content => template('sunetdrive/application/check_nextcloud_mounts.py'),
+      mode    => '0744',
+    }
+    sunet::sudoer {'nagios_run_nextcloud_mounts_command':
+      user_name    => 'nagios',
+      collection   => 'nrpe_nextcloud_mounts_check',
+      command_line => '/usr/lib/nagios/plugins/check_nextcloud_mounts.py'
+    }
+    sunet::nagios::nrpe_command {'check_nextcloud_mounts':
+      command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_nextcloud_mounts.py'
+    }
+  }
 }
diff --git a/manifests/sitemonitornaemon.pp b/manifests/sitemonitornaemon.pp
index 82f2406..b7fadb7 100644
--- a/manifests/sitemonitornaemon.pp
+++ b/manifests/sitemonitornaemon.pp
@@ -99,6 +99,11 @@ class sunetdrive::sitemonitornaemon() {
     description    => 'Status of sarimner interface',
     contact_groups => ['alerts']
   }
-
+  nagioscfg::service {'check_nextcloud_mounts':
+    hostgroup_name => ['sunetdrive::application','sunetdrive::multinode'],
+    check_command  => 'check_nrpe_1arg!check_nextcloud_mounts',
+    description    => 'Nextcloud bucket mounted multiple times',
+    contact_groups => ['alerts']
+  }
 }
 
diff --git a/templates/application/check_nextcloud_mounts.py b/templates/application/check_nextcloud_mounts.py
new file mode 100644
index 0000000..cd304e6
--- /dev/null
+++ b/templates/application/check_nextcloud_mounts.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+from collections import Counter
+import json
+import shlex
+import subprocess
+import sys
+
+buckets = []
+bucket_count = {}
+containers = {}
+exit = 0
+
+get_containers = subprocess.Popen('/usr/local/bin/get_containers', stdout=subprocess.PIPE).stdout.read()
+containers = get_containers.decode().splitlines()
+
+for container in containers:
+    list_command = f"/usr/local/bin/nocc {container} files_external:list --all --show-password --output json"
+    command = shlex.split(list_command)
+    mount_data_byte = subprocess.Popen(command, stdout=subprocess.PIPE).stdout.read()
+    mount_data = json.loads(mount_data_byte.decode())
+    for items in mount_data:
+        buckets.append(items["configuration"]["bucket"])
+    bucket_count = dict(Counter(buckets))
+    for i, (k, v) in enumerate(bucket_count.items()):
+        if v > 1:
+            if i == 0:
+                print("WARNING: buckets with multiple mounts")
+                print(f"bucket {k} is mounted {v} times | {k}_num_mounts={v}")
+            else:
+                print(f"bucket {k} is mounted {v} times | {k}_num_mounts={v}")
+#            lets do exit 0 for now
+#            exit = 1
+sys.exit(exit)

From 17d8d8b2de3c21376452299a9a8e2b07e17132a8 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Wed, 5 Mar 2025 13:21:56 +0100
Subject: [PATCH 02/12] changed wording

---
 manifests/sitemonitornaemon.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manifests/sitemonitornaemon.pp b/manifests/sitemonitornaemon.pp
index b7fadb7..0e286b4 100644
--- a/manifests/sitemonitornaemon.pp
+++ b/manifests/sitemonitornaemon.pp
@@ -102,7 +102,7 @@ class sunetdrive::sitemonitornaemon() {
   nagioscfg::service {'check_nextcloud_mounts':
     hostgroup_name => ['sunetdrive::application','sunetdrive::multinode'],
     check_command  => 'check_nrpe_1arg!check_nextcloud_mounts',
-    description    => 'Nextcloud bucket mounted multiple times',
+    description    => 'S3 bucket(s) with multiple Nextcloud mounts',
     contact_groups => ['alerts']
   }
 }

From e27cf349876caf2c68fad308d2f10d5fdb52d23f Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Wed, 5 Mar 2025 13:28:33 +0100
Subject: [PATCH 03/12] changed wording

---
 manifests/sitemonitornaemon.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manifests/sitemonitornaemon.pp b/manifests/sitemonitornaemon.pp
index 0e286b4..55fc404 100644
--- a/manifests/sitemonitornaemon.pp
+++ b/manifests/sitemonitornaemon.pp
@@ -102,7 +102,7 @@ class sunetdrive::sitemonitornaemon() {
   nagioscfg::service {'check_nextcloud_mounts':
     hostgroup_name => ['sunetdrive::application','sunetdrive::multinode'],
     check_command  => 'check_nrpe_1arg!check_nextcloud_mounts',
-    description    => 'S3 bucket(s) with multiple Nextcloud mounts',
+    description    => 'S3 buckets with multiple Nextcloud mounts',
     contact_groups => ['alerts']
   }
 }

From 0e189ad0efc8d2f893d01e66b5efd7dba63f6fbc Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Wed, 5 Mar 2025 14:15:45 +0100
Subject: [PATCH 04/12] change check_nextcloud_mounts.py output

---
 templates/application/check_nextcloud_mounts.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/templates/application/check_nextcloud_mounts.py b/templates/application/check_nextcloud_mounts.py
index cd304e6..6edde37 100644
--- a/templates/application/check_nextcloud_mounts.py
+++ b/templates/application/check_nextcloud_mounts.py
@@ -10,6 +10,8 @@ buckets = []
 bucket_count = {}
 containers = {}
 exit = 0
+base_message = "OK: no duplicate mounts"
+perf_data = ""
 
 get_containers = subprocess.Popen('/usr/local/bin/get_containers', stdout=subprocess.PIPE).stdout.read()
 containers = get_containers.decode().splitlines()
@@ -22,13 +24,11 @@ for container in containers:
     for items in mount_data:
         buckets.append(items["configuration"]["bucket"])
     bucket_count = dict(Counter(buckets))
-    for i, (k, v) in enumerate(bucket_count.items()):
+    for k, v in bucket_count.items():
         if v > 1:
-            if i == 0:
-                print("WARNING: buckets with multiple mounts")
-                print(f"bucket {k} is mounted {v} times | {k}_num_mounts={v}")
-            else:
-                print(f"bucket {k} is mounted {v} times | {k}_num_mounts={v}")
+            base_message = "WARNING: buckets with multiple mounts |"
+            perf_data += f" {k}={v}"
 #            lets do exit 0 for now
 #            exit = 1
+print(base_message + perf_data)
 sys.exit(exit)

From 8f090bfcac3501dbdaff3607e23a6f406c73893f Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Wed, 5 Mar 2025 14:58:01 +0100
Subject: [PATCH 05/12] testing if $is_multinode is the problem

---
 manifests/app_type.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manifests/app_type.pp b/manifests/app_type.pp
index 521bfbf..9df19aa 100644
--- a/manifests/app_type.pp
+++ b/manifests/app_type.pp
@@ -388,7 +388,7 @@ define sunetdrive::app_type (
       }
     }
   }
-  if $facts['networking']['hostname'] == 'node1' or $is_multinode {
+  if $facts['networking']['hostname'] == 'node1' or $facts['networking']['hostname'] =~ /^multinode[1-9]+\..*/{
     file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py':
       ensure  => present,
       force   => true,

From 3579682de8ad56a5a320331c4998173a7eb63b25 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Wed, 5 Mar 2025 15:00:14 +0100
Subject: [PATCH 06/12] testing if $is_multinode is the problem - correcter
 regex

---
 manifests/app_type.pp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manifests/app_type.pp b/manifests/app_type.pp
index 9df19aa..a674929 100644
--- a/manifests/app_type.pp
+++ b/manifests/app_type.pp
@@ -388,7 +388,7 @@ define sunetdrive::app_type (
       }
     }
   }
-  if $facts['networking']['hostname'] == 'node1' or $facts['networking']['hostname'] =~ /^multinode[1-9]+\..*/{
+  if $facts['networking']['hostname'] == 'node1' or $facts['networking']['hostname'] =~ /^multinode[1-9]+$/{
     file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py':
       ensure  => present,
       force   => true,

From e150fa3625362fe05c7877768a9a7b97d8d803d5 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Wed, 5 Mar 2025 16:22:29 +0100
Subject: [PATCH 07/12] separating multinode from app_type

---
 manifests/app_type.pp  | 35 +++++++++++++++++------------------
 manifests/multinode.pp | 15 +++++++++++++++
 2 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/manifests/app_type.pp b/manifests/app_type.pp
index a674929..155f93e 100644
--- a/manifests/app_type.pp
+++ b/manifests/app_type.pp
@@ -101,6 +101,23 @@ define sunetdrive::app_type (
       content => template('sunetdrive/application/get_containers'),
       mode    => '0744',
     }
+    if ($nodenumber == 3) {
+      file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py':
+        ensure  => present,
+        owner   => 'root',
+        group   => 'root',
+        content => template('sunetdrive/application/check_nextcloud_mounts.py'),
+        mode    => '0744',
+      }
+      sunet::sudoer {'nagios_run_nextcloud_mounts_command':
+        user_name    => 'nagios',
+        collection   => 'nrpe_nextcloud_mounts_check',
+        command_line => '/usr/lib/nagios/plugins/check_nextcloud_mounts.py'
+      }
+      sunet::nagios::nrpe_command {'check_nextcloud_mounts':
+        command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_nextcloud_mounts.py'
+      }
+    }
     if ($nodenumber == 3) {
       file { '/usr/local/bin/scan_external_mounts':
         ensure  => present,
@@ -388,22 +405,4 @@ define sunetdrive::app_type (
       }
     }
   }
-  if $facts['networking']['hostname'] == 'node1' or $facts['networking']['hostname'] =~ /^multinode[1-9]+$/{
-    file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py':
-      ensure  => present,
-      force   => true,
-      owner   => 'root',
-      group   => 'root',
-      content => template('sunetdrive/application/check_nextcloud_mounts.py'),
-      mode    => '0744',
-    }
-    sunet::sudoer {'nagios_run_nextcloud_mounts_command':
-      user_name    => 'nagios',
-      collection   => 'nrpe_nextcloud_mounts_check',
-      command_line => '/usr/lib/nagios/plugins/check_nextcloud_mounts.py'
-    }
-    sunet::nagios::nrpe_command {'check_nextcloud_mounts':
-      command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_nextcloud_mounts.py'
-    }
-  }
 }
diff --git a/manifests/multinode.pp b/manifests/multinode.pp
index 7cf12b8..90cc651 100644
--- a/manifests/multinode.pp
+++ b/manifests/multinode.pp
@@ -46,6 +46,21 @@ class sunetdrive::multinode (
     content => template('sunetdrive/application/get_containers'),
     mode    => '0744',
   }
+  file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    content => template('sunetdrive/application/check_nextcloud_mounts.py'),
+    mode    => '0744',
+  }
+  sunet::sudoer {'nagios_run_nextcloud_mounts_command':
+    user_name    => 'nagios',
+    collection   => 'nrpe_nextcloud_mounts_check',
+    command_line => '/usr/lib/nagios/plugins/check_nextcloud_mounts.py'
+  }
+  sunet::nagios::nrpe_command {'check_nextcloud_mounts':
+    command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_nextcloud_mounts.py'
+  }
   file { '/usr/local/bin/scan_external_mounts':
     ensure  => present,
     force   => true,

From 4b952bac402b94c2f79148fb777c4b4583519094 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Thu, 6 Mar 2025 15:05:00 +0100
Subject: [PATCH 08/12] trying to improve output of check_nextcloud_mounts
 script

---
 .../application/check_nextcloud_mounts.py     | 23 +++++++++++++------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/templates/application/check_nextcloud_mounts.py b/templates/application/check_nextcloud_mounts.py
index 6edde37..05fd9ff 100644
--- a/templates/application/check_nextcloud_mounts.py
+++ b/templates/application/check_nextcloud_mounts.py
@@ -6,29 +6,38 @@ import shlex
 import subprocess
 import sys
 
+exit = 0
 buckets = []
 bucket_count = {}
 containers = {}
-exit = 0
 base_message = "OK: no duplicate mounts"
-perf_data = ""
+long_message = ""
 
 get_containers = subprocess.Popen('/usr/local/bin/get_containers', stdout=subprocess.PIPE).stdout.read()
 containers = get_containers.decode().splitlines()
 
-for container in containers:
+for i, container in enumerate(containers, start=1):
     list_command = f"/usr/local/bin/nocc {container} files_external:list --all --show-password --output json"
     command = shlex.split(list_command)
     mount_data_byte = subprocess.Popen(command, stdout=subprocess.PIPE).stdout.read()
-    mount_data = json.loads(mount_data_byte.decode())
+    try:
+        mount_data = json.loads(mount_data_byte.decode())
+    except json.decoder.JSONDecodeError as err:
+        if i == 1 or i != len(containers):
+            base_message = "WARNING: invalid json"
+        long_message +=  f"\ncontainer: {container} - json decode error: {err}"
+#        lets do exit 0 for now
+#        exit = 1
+        continue
     for items in mount_data:
         buckets.append(items["configuration"]["bucket"])
     bucket_count = dict(Counter(buckets))
     for k, v in bucket_count.items():
         if v > 1:
-            base_message = "WARNING: buckets with multiple mounts |"
-            perf_data += f" {k}={v}"
+            base_message = "WARNING: buckets with multiple mounts"
+            long_message += f"\ncontainer: {container} - bucket: {k} - {v}"
 #            lets do exit 0 for now
 #            exit = 1
-print(base_message + perf_data)
+print(base_message)
+print(long_message.lstrip())
 sys.exit(exit)

From 0b3a2d43cba123b6693f980de1d4ced723b9df92 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Thu, 6 Mar 2025 15:42:32 +0100
Subject: [PATCH 09/12] buckets needs to be reset each loop

---
 templates/application/check_nextcloud_mounts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/application/check_nextcloud_mounts.py b/templates/application/check_nextcloud_mounts.py
index 05fd9ff..8e33300 100644
--- a/templates/application/check_nextcloud_mounts.py
+++ b/templates/application/check_nextcloud_mounts.py
@@ -7,7 +7,6 @@ import subprocess
 import sys
 
 exit = 0
-buckets = []
 bucket_count = {}
 containers = {}
 base_message = "OK: no duplicate mounts"
@@ -17,6 +16,7 @@ get_containers = subprocess.Popen('/usr/local/bin/get_containers', stdout=subpro
 containers = get_containers.decode().splitlines()
 
 for i, container in enumerate(containers, start=1):
+    buckets = []
     list_command = f"/usr/local/bin/nocc {container} files_external:list --all --show-password --output json"
     command = shlex.split(list_command)
     mount_data_byte = subprocess.Popen(command, stdout=subprocess.PIPE).stdout.read()

From 1aea2f56742dfa6cc51af9ffab9fe68410e3e2e4 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Thu, 6 Mar 2025 16:26:05 +0100
Subject: [PATCH 10/12] apparently "You do not need to declare variables before
 using them" in python - cleanup

---
 templates/application/check_nextcloud_mounts.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/templates/application/check_nextcloud_mounts.py b/templates/application/check_nextcloud_mounts.py
index 8e33300..5a7cb7a 100644
--- a/templates/application/check_nextcloud_mounts.py
+++ b/templates/application/check_nextcloud_mounts.py
@@ -7,8 +7,6 @@ import subprocess
 import sys
 
 exit = 0
-bucket_count = {}
-containers = {}
 base_message = "OK: no duplicate mounts"
 long_message = ""
 

From d5c3486425e51d797ffec13c54058951fb733ce3 Mon Sep 17 00:00:00 2001
From: Rikard Danielsson <richir@sunet.se>
Date: Fri, 7 Mar 2025 08:35:33 +0100
Subject: [PATCH 11/12] don't print empty line

---
 templates/application/check_nextcloud_mounts.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/templates/application/check_nextcloud_mounts.py b/templates/application/check_nextcloud_mounts.py
index 5a7cb7a..124a6f6 100644
--- a/templates/application/check_nextcloud_mounts.py
+++ b/templates/application/check_nextcloud_mounts.py
@@ -37,5 +37,6 @@ for i, container in enumerate(containers, start=1):
 #            lets do exit 0 for now
 #            exit = 1
 print(base_message)
-print(long_message.lstrip())
+if long_message != "":
+    print(long_message.lstrip())
 sys.exit(exit)

From d010d27f20dafd73632121c47366eae3030cade3 Mon Sep 17 00:00:00 2001
From: Micke Nordin <kano@sunet.se>
Date: Wed, 12 Mar 2025 11:44:08 +0100
Subject: [PATCH 12/12] Let's gooooo

---
 files/scriptreciver/sysctl-d-gofasta.conf |  6 ++++++
 manifests/scriptreceiver.pp               | 25 +++++++++++++++++++++--
 2 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 files/scriptreciver/sysctl-d-gofasta.conf

diff --git a/files/scriptreciver/sysctl-d-gofasta.conf b/files/scriptreciver/sysctl-d-gofasta.conf
new file mode 100644
index 0000000..678f5fb
--- /dev/null
+++ b/files/scriptreciver/sysctl-d-gofasta.conf
@@ -0,0 +1,6 @@
+net.core.rmem_max=67108864
+net.core.wmem_max=67108864
+net.ipv4.tcp_rmem="4096 87380 33554432"
+net.ipv4.tcp_wmem="4096 87380 33554432"
+net.core.default_qdisc=fq
+net.ipv4.tcp_congestion_control=bbr
diff --git a/manifests/scriptreceiver.pp b/manifests/scriptreceiver.pp
index 6aca740..923cc38 100644
--- a/manifests/scriptreceiver.pp
+++ b/manifests/scriptreceiver.pp
@@ -5,7 +5,19 @@ class sunetdrive::scriptreceiver()
   sunet::system_user {'script': username => 'script', group => 'script', managehome => true, shell => '/bin/bash' }
 
   # These tasks correspond to a ${task}.erb.sh template
-  $tasks = ['list_users', 'list_files_for_user', 'create_bucket', 'backup_db', 'purge_backups', 'maintenancemode', 'restart_sunet_service', 'start_sentinel', 'stop_sentinel', 'removeswap', 'backup_multinode_db']
+  $tasks = [
+    'list_users',
+    'list_files_for_user',
+    'create_bucket',
+    'backup_db',
+    'purge_backups',
+    'maintenancemode',
+    'restart_sunet_service',
+    'start_sentinel',
+    'stop_sentinel',
+    'removeswap',
+    'backup_multinode_db'
+  ]
 
   $environment = sunetdrive::get_environment()
   $config = hiera_hash($environment)
@@ -35,7 +47,16 @@ class sunetdrive::scriptreceiver()
     type   => 'ssh-ed25519',
     key    => $script_pub_key,
   }
-
+  file { '/etc/sysctl.d/gofasta.conf':
+    content => file('sunetdrive/scriptreceiver/systctl-d-gofasta.conf'),
+    mode    => '0644',
+  }
+  -> exec { 'gofasta_with_sysctl':
+    command     => 'sysctl -p /etc/sysctl.d/gofasta.conf',
+    path        => ['/bin','/usr/bin','/sbin','/usr/sbin'],
+    subscribe   => File['/etc/sysctl.d/gofasta.conf'],
+    refreshonly => true,
+  }
   file { '/opt/rotate':
     ensure => directory,
     mode   => '0750',