From d1de5d052d827f4970fe8e9c5229b0ab8001629a Mon Sep 17 00:00:00 2001 From: Patrik Holmqvist Date: Thu, 27 Mar 2025 08:44:57 +0100 Subject: [PATCH] Attempt to simplify telegraf setup, SC-2522 --- .../puppet/modules/eid/manifests/telegraf2.pp | 51 ++++++++++-------- .../telegraf2/docker-compose.yml.erb | 8 --- .../eid/templates/telegraf2/telegraf.conf.erb | 53 +++++++++++++++++++ .../overlay/etc/hiera/data/local.eyaml | 1 - 4 files changed, 83 insertions(+), 30 deletions(-) create mode 100644 global/overlay/etc/puppet/modules/eid/templates/telegraf2/telegraf.conf.erb diff --git a/global/overlay/etc/puppet/modules/eid/manifests/telegraf2.pp b/global/overlay/etc/puppet/modules/eid/manifests/telegraf2.pp index 46b69710..95854ae8 100644 --- a/global/overlay/etc/puppet/modules/eid/manifests/telegraf2.pp +++ b/global/overlay/etc/puppet/modules/eid/manifests/telegraf2.pp @@ -28,31 +28,40 @@ class eid::telegraf2 ( description => 'telegraf' } - file {["${service_dir}/telegraf.conf.d"]: ensure => directory } - concat {"${service_dir}/telegraf.conf": - owner => 'root', - group => 'root', - mode => '0644', + #file {["${service_dir}/telegraf.conf.d"]: ensure => directory } + #concat {"${service_dir}/telegraf.conf": + # owner => 'root', + # group => 'root', + # mode => '0644', #notify => [Sunet::Docker_compose["telegraf"]] - } + #} # Below not pretty but I'm piggy-backing on this class to add a secondary output to test the new influx2-server - if ($influx2_token != 'NOT_SET_IN_HIERA') { - file { "${service_dir}/telegraf.conf.d/95-telegraf.conf": - ensure => file, - owner => 'root', - group => 'root', - path => "${service_dir}/telegraf.conf.d/95-telegraf.conf", - mode => '0644', - content => template('eid/telegraf2/95-telegraf.conf.erb'), - } - } + #if ($influx2_token != 'NOT_SET_IN_HIERA') { + # file { "${service_dir}/telegraf.conf.d/95-telegraf.conf": + # ensure => file, + # owner => 'root', + # group => 'root', + # path => "${service_dir}/telegraf.conf.d/95-telegraf.conf", + # mode => '0644', + # content => template('eid/telegraf2/95-telegraf.conf.erb'), + # } + #} - eid_glob(["${service_dir}/telegraf.conf.d/*.conf"]).each |String $file| { - concat::fragment { $file: - target => "${service_dir}/telegraf.conf", - source => $file, - } + #eid_glob(["${service_dir}/telegraf.conf.d/*.conf"]).each |String $file| { + # concat::fragment { $file: + # target => "${service_dir}/telegraf.conf", + # source => $file, + # } + #} + + file { "${service_dir}/telegraf.conf.d/telegraf.conf": + ensure => file, + owner => 'root', + group => 'root', + path => "${service_dir}/telegraf.conf.d/telegraf.conf", + mode => '0644', + content => template('eid/telegraf2/telegraf.conf.erb'), } } diff --git a/global/overlay/etc/puppet/modules/eid/templates/telegraf2/docker-compose.yml.erb b/global/overlay/etc/puppet/modules/eid/templates/telegraf2/docker-compose.yml.erb index eaa0d6ce..138f5105 100644 --- a/global/overlay/etc/puppet/modules/eid/templates/telegraf2/docker-compose.yml.erb +++ b/global/overlay/etc/puppet/modules/eid/templates/telegraf2/docker-compose.yml.erb @@ -3,14 +3,6 @@ services: telegraf: image: docker.sunet.se/eduid/telegraf:<%= @version %> container_name: telegraf - environment: - - INFLUXDB2_TOKEN=<%= @influxdb2_token %> - - INFLUXDB2_ORG=<%= @influxdb2_org %> - - INFLUXDB2_BUCKET=<%= @influxdb2_bucket %> - - INFLUXDB2_BUCKET_PROXY=<%= @influxdb2_bucket_proxy %> -<% if @influx2_token != 'NOT_SET_IN_HIERA' -%> - - INFLUX2_TOKEN=<%= @influx2_token %> -<% end -%> hostname: <%= @server_fqdn %> diff --git a/global/overlay/etc/puppet/modules/eid/templates/telegraf2/telegraf.conf.erb b/global/overlay/etc/puppet/modules/eid/templates/telegraf2/telegraf.conf.erb new file mode 100644 index 00000000..86ea17f7 --- /dev/null +++ b/global/overlay/etc/puppet/modules/eid/templates/telegraf2/telegraf.conf.erb @@ -0,0 +1,53 @@ +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logfile = "" + hostname = "" + omit_hostname = false +[[inputs.system]] + +[[inputs.tail]] + files = ["/var/log/eidas_stats.log"] + data_format = "json" + tag_keys = ["requester","country"] + json_name_key = "type" + json_time_key = "timestamp" + json_time_format = "unix_ms" + json_timezone = "Local" +[[inputs.tail]] + files = ["/var/log/eidas_proxy.log"] + data_format = "json" + json_name_key = "type" + tag_keys = ["type", "principal", "data_eidasResponseLoA", "data_requesterId", "data_requestedLoa", "data_connectorCuntry", "data_connectorEntityId", "data_nationalIdP", "data_loaMatching", "data_spType", "data_eIDASAssertionIssueTime", "data_requestedLoa" ] + [inputs.tail.tags] + influxdb_database = "proxy" +[[processors.parser]] + parse_fields = ["message"] + merge = "override" + data_format = "json" + +[[outputs.influxdb_v2]] + urls = <%= $influx_urls %> + token = <%= @influx2_token %> + organization = "SwedenConnect" + bucket = "eidas" + insecure_skip_verify = true + [outputs.influxdb_v2.tagdrop] + influxdb_database = ["*"] +[[outputs.influxdb_v2]] + urls = <%= $influx_urls %> + token = <%= @influx2_token %> + organization = "SwedenConnect" + bucket = "eidas-proxy" + insecure_skip_verify = true + tagexclude = ["influxdb_database"] + [outputs.influxdb_v2.tagpass] + influxdb_database = ["proxy"] diff --git a/log-test-sto3-1.komreg.net/overlay/etc/hiera/data/local.eyaml b/log-test-sto3-1.komreg.net/overlay/etc/hiera/data/local.eyaml index ca64ed51..a3bb6562 100644 --- a/log-test-sto3-1.komreg.net/overlay/etc/hiera/data/local.eyaml +++ b/log-test-sto3-1.komreg.net/overlay/etc/hiera/data/local.eyaml @@ -1,3 +1,2 @@ --- -influxdb2_token: ENC[PKCS7,MIIDLwYJKoZIhvcNAQcDoIIDIDCCAxwCAQAxggKGMIICggIBADBqMFIxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxIzAhBgNVBAMMGmxvZy10ZXN0LXN0bzMtMS5rb21yZWcubmV0AhRdaHNQ11EJCml8OOzq2F07dXaxqjANBgkqhkiG9w0BAQEFAASCAgCYx3n0XR54zw46za2BxY2HmHnT+CCRfQercNnC0bZIA0HHYg5aEsDCy3yyY2YFf7ChAzgklHV4fa4pNo+1TQuq2G5QjOwuTNaBjOjRCTnFPhfVep+DY8ab/FY1B/NmMHrCEpZbhy2X9LRY2GH5wAgZQDRq3khiI1d6bJQmB0hySv1N7GHGmb2nnRFMDyZP3H4T5nKePeBNidDZfe7RLDE1QplcqAPBkOay7THAe/IarKmBjQQ76V7s+p7QVy5uvktTCFAM/9+yf0FaYojRFWktnCScEwXVkB+a0+CsJ4wDSel+8UZEagyrUv0nlyfgQJIWDWLt/14zCaCeYTz1QjIccj9IJNrNkkyFOudrR2dT5kcbNagzIukV5IdZrFL8IVUcllAPXHrAXYIbl6dCkCC6SCl6Vzrf+murGRIYiG5MCcXIdUB1lIZ9+v35l5p3QA6ATDINthyVbIlPx0rLCk1sjWf1TGioa9iqIUDTPIckV4ONKRy4SUK3E5BDPO+LsU6xx02JhqebhSp3ZWN7BadZUApVmX5dOshk41KlYK6EdTumLD/bM4/zmVwUne4qEJ94YXK+vb8m5F0xHAWdEH1+aj6NzgOMBFWvSGLbWLKx2961sFw73HyE8cLhu5LzG45dz1hSy0AcRyi/6+iZa4CfgG2nU4qKBCRxsPa5unJTJDCBjAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQJjMYrYAPfqZUG8Por7RBhoBgmg0ICZHH6dkA1cYRNAWS0gRlOBnvgR8KfGwjyb3UqBH7VqXvA2BOWQPEwWGzgjMmRt+48y593TcKfK3TWJ8xIfdG74N7vI8mDkjc7IFgWgQ/bbD1m7FW/9twczvHf5zg] influx2_token: ENC[PKCS7,MIIDLwYJKoZIhvcNAQcDoIIDIDCCAxwCAQAxggKGMIICggIBADBqMFIxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxIzAhBgNVBAMMGmxvZy10ZXN0LXN0bzMtMS5rb21yZWcubmV0AhRdaHNQ11EJCml8OOzq2F07dXaxqjANBgkqhkiG9w0BAQEFAASCAgC6Z66UDn0xrfaTFJpZFu9+GnO6PToZns+21IcwIaFSXrX/pe7tsXCp5WV+1+McEe57MIln0SYip5mukbtZvHAxMlrOa6YDP4rFYmxQ3AQbOTtomgU0KkH6P8nCQF49pwm7S93nLGcqG0l3h/aj5mhe4D4rWXVN+xbMzaWg2h/dmngCc6SsWkP/iB52GsoEldr0Bq9wsJwkMQZn9eTL7Q3OqhvxQwvQhPdQbOcq/IQTwZDaIZUZyoNfmuUR6uYb3Xvwh6H8+dfQSDFYMjF9WdwNVUI7Y4hT1l3eOpuknJtm8znSOpUrj0Urtx4PLq+ah8eJeKyhSF1iNZkQKH2h48aHPBHz6ndT/pibF89lQVK7WaDuuFDLrkYyh7sAi5EJTaLzjUD7WK1E4VZSs+fT0G4QxCvKBPdAYqRn4ZbHZPeXF+NUcjiMr1Wa+RvBqWDYtz8YJa9ntb3637VvhXBG+HZHhB7rWGa1IqJNZ2y+LQOjnMcdVO1Z92e4RVRqWp1G9o9KTrmBSrQpNY2Tf4/beWc77ieWJ5zrYGAung7EFK9/TkA5qxZVGDvmtLqKiwRB9+Pq/sGqqMTh+ypbdSr9uEmpLqf3SWP/8TZP7zrFROjTiWpZo8s3vuTXainXi4bmaUAmQOjCGrgUKt7/srBbxBqP1iUrrEL6x1XPur/TtrUZUDCBjAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQE7L+6e5g2iCsZcbgQenYN4BgLQKTV3BQVSXj7D1xiB3J5LRDLZFZtMuliUr9SDjLNSXWRy7M2FpoMMvQvaQxKrR4Mb9lZqKbJTtoaUBjLP627MYUrGguMjV7T1nxI4Q/ATZLPV3/s0qdDo0ZuR1pkvjf]