diff --git a/files/scriptreciver/sysctl-d-gofasta.conf b/files/scriptreciver/sysctl-d-gofasta.conf new file mode 100644 index 0000000..dc688c5 --- /dev/null +++ b/files/scriptreciver/sysctl-d-gofasta.conf @@ -0,0 +1,6 @@ +net.core.rmem_max=67108864 +net.core.wmem_max=67108864 +net.ipv4.tcp_rmem=4096 87380 33554432 +net.ipv4.tcp_wmem=4096 87380 33554432 +net.core.default_qdisc=fq +net.ipv4.tcp_congestion_control=bbr diff --git a/manifests/app_type.pp b/manifests/app_type.pp index 1996271..155f93e 100644 --- a/manifests/app_type.pp +++ b/manifests/app_type.pp @@ -101,6 +101,23 @@ define sunetdrive::app_type ( content => template('sunetdrive/application/get_containers'), mode => '0744', } + if ($nodenumber == 3) { + file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py': + ensure => present, + owner => 'root', + group => 'root', + content => template('sunetdrive/application/check_nextcloud_mounts.py'), + mode => '0744', + } + sunet::sudoer {'nagios_run_nextcloud_mounts_command': + user_name => 'nagios', + collection => 'nrpe_nextcloud_mounts_check', + command_line => '/usr/lib/nagios/plugins/check_nextcloud_mounts.py' + } + sunet::nagios::nrpe_command {'check_nextcloud_mounts': + command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_nextcloud_mounts.py' + } + } if ($nodenumber == 3) { file { '/usr/local/bin/scan_external_mounts': ensure => present, @@ -388,5 +405,4 @@ define sunetdrive::app_type ( } } } - } diff --git a/manifests/multinode.pp b/manifests/multinode.pp index 7cf12b8..90cc651 100644 --- a/manifests/multinode.pp +++ b/manifests/multinode.pp @@ -46,6 +46,21 @@ class sunetdrive::multinode ( content => template('sunetdrive/application/get_containers'), mode => '0744', } + file { '/usr/lib/nagios/plugins/check_nextcloud_mounts.py': + ensure => present, + owner => 'root', + group => 'root', + content => template('sunetdrive/application/check_nextcloud_mounts.py'), + mode => '0744', + } + sunet::sudoer {'nagios_run_nextcloud_mounts_command': + user_name => 'nagios', + collection => 'nrpe_nextcloud_mounts_check', + command_line => '/usr/lib/nagios/plugins/check_nextcloud_mounts.py' + } + sunet::nagios::nrpe_command {'check_nextcloud_mounts': + command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_nextcloud_mounts.py' + } file { '/usr/local/bin/scan_external_mounts': ensure => present, force => true, diff --git a/manifests/scriptreceiver.pp b/manifests/scriptreceiver.pp index 6aca740..bae40ba 100644 --- a/manifests/scriptreceiver.pp +++ b/manifests/scriptreceiver.pp @@ -5,7 +5,19 @@ class sunetdrive::scriptreceiver() sunet::system_user {'script': username => 'script', group => 'script', managehome => true, shell => '/bin/bash' } # These tasks correspond to a ${task}.erb.sh template - $tasks = ['list_users', 'list_files_for_user', 'create_bucket', 'backup_db', 'purge_backups', 'maintenancemode', 'restart_sunet_service', 'start_sentinel', 'stop_sentinel', 'removeswap', 'backup_multinode_db'] + $tasks = [ + 'list_users', + 'list_files_for_user', + 'create_bucket', + 'backup_db', + 'purge_backups', + 'maintenancemode', + 'restart_sunet_service', + 'start_sentinel', + 'stop_sentinel', + 'removeswap', + 'backup_multinode_db' + ] $environment = sunetdrive::get_environment() $config = hiera_hash($environment) @@ -35,7 +47,9 @@ class sunetdrive::scriptreceiver() type => 'ssh-ed25519', key => $script_pub_key, } - + file { '/etc/sysctl.d/gofasta.conf': + ensure => 'absent', + } file { '/opt/rotate': ensure => directory, mode => '0750', diff --git a/manifests/sitemonitornaemon.pp b/manifests/sitemonitornaemon.pp index 82f2406..64bdb37 100644 --- a/manifests/sitemonitornaemon.pp +++ b/manifests/sitemonitornaemon.pp @@ -11,6 +11,9 @@ class sunetdrive::sitemonitornaemon() { $environment = sunetdrive::get_environment() $influx_passwd = safe_hiera('influx_passwd') $slack_url = safe_hiera('slack_url') + $extra_host_groups = { + node3_hosts => join($facts['configured_hosts_in_cosmos']['all'].filter |$host| { $host =~ /^node3\./ }, ',') + } file { '/usr/local/bin/slack_nagios.sh': ensure => present, @@ -45,6 +48,11 @@ class sunetdrive::sitemonitornaemon() { content => template('sunetdrive/monitor/sunetdrive_thruk_templates.conf.erb'), mode => '0644', } + file { '/etc/naemon/conf.d/sunetdrive_extra_hostgroups.cfg': + ensure => present, + content => template('sunetdrive/monitor/sunetdrive_extra_hostgroups.cfg.erb'), + mode => '0644', + } nagioscfg::service {'check_scriptherder': hostgroup_name => ['sunetdrive::nrpe'], check_command => 'check_nrpe_1arg_to300!check_scriptherder', @@ -99,6 +107,12 @@ class sunetdrive::sitemonitornaemon() { description => 'Status of sarimner interface', contact_groups => ['alerts'] } + nagioscfg::service {'check_nextcloud_mounts': + hostgroup_name => ['node3_hosts','sunetdrive::multinode'], + check_command => 'check_nrpe_1arg!check_nextcloud_mounts', + description => 'S3 buckets with multiple Nextcloud mounts', + contact_groups => ['alerts'] + } } diff --git a/templates/application/check_nextcloud_mounts.py b/templates/application/check_nextcloud_mounts.py new file mode 100644 index 0000000..124a6f6 --- /dev/null +++ b/templates/application/check_nextcloud_mounts.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +from collections import Counter +import json +import shlex +import subprocess +import sys + +exit = 0 +base_message = "OK: no duplicate mounts" +long_message = "" + +get_containers = subprocess.Popen('/usr/local/bin/get_containers', stdout=subprocess.PIPE).stdout.read() +containers = get_containers.decode().splitlines() + +for i, container in enumerate(containers, start=1): + buckets = [] + list_command = f"/usr/local/bin/nocc {container} files_external:list --all --show-password --output json" + command = shlex.split(list_command) + mount_data_byte = subprocess.Popen(command, stdout=subprocess.PIPE).stdout.read() + try: + mount_data = json.loads(mount_data_byte.decode()) + except json.decoder.JSONDecodeError as err: + if i == 1 or i != len(containers): + base_message = "WARNING: invalid json" + long_message += f"\ncontainer: {container} - json decode error: {err}" +# lets do exit 0 for now +# exit = 1 + continue + for items in mount_data: + buckets.append(items["configuration"]["bucket"]) + bucket_count = dict(Counter(buckets)) + for k, v in bucket_count.items(): + if v > 1: + base_message = "WARNING: buckets with multiple mounts" + long_message += f"\ncontainer: {container} - bucket: {k} - {v}" +# lets do exit 0 for now +# exit = 1 +print(base_message) +if long_message != "": + print(long_message.lstrip()) +sys.exit(exit) diff --git a/templates/monitor/sunetdrive_extra_hostgroups.cfg.erb b/templates/monitor/sunetdrive_extra_hostgroups.cfg.erb new file mode 100644 index 0000000..96b80fc --- /dev/null +++ b/templates/monitor/sunetdrive_extra_hostgroups.cfg.erb @@ -0,0 +1,8 @@ +<% @extra_host_groups.each do |group, members| -%> +# <%= group %> +define hostgroup { + hostgroup_name <%= group %> + alias <%= group %> + members <%= members %> +} +<% end -%>