Attempt to simplify telegraf setup, SC-2522

2025-03-27 08:44:57 +01:00 · 2025-03-27 08:44:57 +01:00 · d1de5d052d
commit d1de5d052d
parent b6c7687b9e
4 changed files with 83 additions and 30 deletions
--- a/global/overlay/etc/puppet/modules/eid/manifests/telegraf2.pp
+++ b/global/overlay/etc/puppet/modules/eid/manifests/telegraf2.pp
@ -28,31 +28,40 @@ class eid::telegraf2 (
    description      => 'telegraf'
  }

-  file {["${service_dir}/telegraf.conf.d"]: ensure => directory }
-  concat {"${service_dir}/telegraf.conf":
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0644',
+  #file {["${service_dir}/telegraf.conf.d"]: ensure => directory }
+  #concat {"${service_dir}/telegraf.conf":
+  #  owner  => 'root',
+  #  group  => 'root',
+  #  mode   => '0644',
    #notify => [Sunet::Docker_compose["telegraf"]]
-  }
+  #}

  # Below not pretty but I'm piggy-backing on this class to add a secondary output to test the new influx2-server
-  if ($influx2_token != 'NOT_SET_IN_HIERA') {
-    file { "${service_dir}/telegraf.conf.d/95-telegraf.conf":
+  #if ($influx2_token != 'NOT_SET_IN_HIERA') {
+  #  file { "${service_dir}/telegraf.conf.d/95-telegraf.conf":
+  #    ensure  => file,
+  #    owner   => 'root',
+  #    group   => 'root',
+  #    path    => "${service_dir}/telegraf.conf.d/95-telegraf.conf",
+  #    mode    => '0644',
+  #    content => template('eid/telegraf2/95-telegraf.conf.erb'),
+  #  }
+  #}
+
+  #eid_glob(["${service_dir}/telegraf.conf.d/*.conf"]).each |String $file| {
+  #  concat::fragment { $file:
+  #    target => "${service_dir}/telegraf.conf",
+  #    source => $file,
+  #  }
+  #}
+
+  file { "${service_dir}/telegraf.conf.d/telegraf.conf":
    ensure  => file,
    owner   => 'root',
    group   => 'root',
-      path    => "${service_dir}/telegraf.conf.d/95-telegraf.conf",
+    path    => "${service_dir}/telegraf.conf.d/telegraf.conf",
    mode    => '0644',
-      content => template('eid/telegraf2/95-telegraf.conf.erb'),
-    }
-  }
-
-  eid_glob(["${service_dir}/telegraf.conf.d/*.conf"]).each |String $file| {
-    concat::fragment { $file:
-      target => "${service_dir}/telegraf.conf",
-      source => $file,
-    }
+    content => template('eid/telegraf2/telegraf.conf.erb'),
  }

 }
--- a/global/overlay/etc/puppet/modules/eid/templates/telegraf2/docker-compose.yml.erb
+++ b/global/overlay/etc/puppet/modules/eid/templates/telegraf2/docker-compose.yml.erb
@ -3,14 +3,6 @@ services:
  telegraf:
    image: docker.sunet.se/eduid/telegraf:<%= @version %>
    container_name: telegraf
-    environment:
-      - INFLUXDB2_TOKEN=<%= @influxdb2_token %>
-      - INFLUXDB2_ORG=<%= @influxdb2_org %>
-      - INFLUXDB2_BUCKET=<%= @influxdb2_bucket %>
-      - INFLUXDB2_BUCKET_PROXY=<%= @influxdb2_bucket_proxy %>
-<% if @influx2_token != 'NOT_SET_IN_HIERA' -%>
-      - INFLUX2_TOKEN=<%= @influx2_token %>
-<% end -%>

    hostname: <%= @server_fqdn %>

--- a/global/overlay/etc/puppet/modules/eid/templates/telegraf2/telegraf.conf.erb
+++ b/global/overlay/etc/puppet/modules/eid/templates/telegraf2/telegraf.conf.erb
@ -0,0 +1,53 @@
+[agent]
+  interval = "10s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  debug = false
+  quiet = false
+  logfile = ""
+  hostname = ""
+  omit_hostname = false
+[[inputs.system]]
+
+[[inputs.tail]]
+  files = ["/var/log/eidas_stats.log"]
+  data_format = "json"
+  tag_keys = ["requester","country"]
+  json_name_key = "type"
+  json_time_key = "timestamp"
+  json_time_format = "unix_ms"
+  json_timezone = "Local"
+[[inputs.tail]]
+  files = ["/var/log/eidas_proxy.log"]
+  data_format = "json"
+  json_name_key = "type"
+  tag_keys = ["type", "principal", "data_eidasResponseLoA", "data_requesterId", "data_requestedLoa", "data_connectorCuntry", "data_connectorEntityId", "data_nationalIdP", "data_loaMatching", "data_spType", "data_eIDASAssertionIssueTime", "data_requestedLoa" ]
+  [inputs.tail.tags]
+    influxdb_database = "proxy"
+[[processors.parser]]
+  parse_fields = ["message"]
+  merge = "override"
+  data_format = "json"
+
+[[outputs.influxdb_v2]]
+  urls = <%= $influx_urls %>
+  token = <%= @influx2_token %>
+  organization = "SwedenConnect"
+  bucket = "eidas"
+  insecure_skip_verify = true
+  [outputs.influxdb_v2.tagdrop]
+    influxdb_database = ["*"]
+[[outputs.influxdb_v2]]
+  urls = <%= $influx_urls %>
+  token = <%= @influx2_token %>
+  organization = "SwedenConnect"
+  bucket = "eidas-proxy"
+  insecure_skip_verify = true
+  tagexclude = ["influxdb_database"]
+  [outputs.influxdb_v2.tagpass]
+    influxdb_database = ["proxy"]
--- a/log-test-sto3-1.komreg.net/overlay/etc/hiera/data/local.eyaml
+++ b/log-test-sto3-1.komreg.net/overlay/etc/hiera/data/local.eyaml
@ -1,3 +1,2 @@
 ---
-influxdb2_token: ENC[PKCS7,MIIDLwYJKoZIhvcNAQcDoIIDIDCCAxwCAQAxggKGMIICggIBADBqMFIxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxIzAhBgNVBAMMGmxvZy10ZXN0LXN0bzMtMS5rb21yZWcubmV0AhRdaHNQ11EJCml8OOzq2F07dXaxqjANBgkqhkiG9w0BAQEFAASCAgCYx3n0XR54zw46za2BxY2HmHnT+CCRfQercNnC0bZIA0HHYg5aEsDCy3yyY2YFf7ChAzgklHV4fa4pNo+1TQuq2G5QjOwuTNaBjOjRCTnFPhfVep+DY8ab/FY1B/NmMHrCEpZbhy2X9LRY2GH5wAgZQDRq3khiI1d6bJQmB0hySv1N7GHGmb2nnRFMDyZP3H4T5nKePeBNidDZfe7RLDE1QplcqAPBkOay7THAe/IarKmBjQQ76V7s+p7QVy5uvktTCFAM/9+yf0FaYojRFWktnCScEwXVkB+a0+CsJ4wDSel+8UZEagyrUv0nlyfgQJIWDWLt/14zCaCeYTz1QjIccj9IJNrNkkyFOudrR2dT5kcbNagzIukV5IdZrFL8IVUcllAPXHrAXYIbl6dCkCC6SCl6Vzrf+murGRIYiG5MCcXIdUB1lIZ9+v35l5p3QA6ATDINthyVbIlPx0rLCk1sjWf1TGioa9iqIUDTPIckV4ONKRy4SUK3E5BDPO+LsU6xx02JhqebhSp3ZWN7BadZUApVmX5dOshk41KlYK6EdTumLD/bM4/zmVwUne4qEJ94YXK+vb8m5F0xHAWdEH1+aj6NzgOMBFWvSGLbWLKx2961sFw73HyE8cLhu5LzG45dz1hSy0AcRyi/6+iZa4CfgG2nU4qKBCRxsPa5unJTJDCBjAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQJjMYrYAPfqZUG8Por7RBhoBgmg0ICZHH6dkA1cYRNAWS0gRlOBnvgR8KfGwjyb3UqBH7VqXvA2BOWQPEwWGzgjMmRt+48y593TcKfK3TWJ8xIfdG74N7vI8mDkjc7IFgWgQ/bbD1m7FW/9twczvHf5zg]
 influx2_token: ENC[PKCS7,MIIDLwYJKoZIhvcNAQcDoIIDIDCCAxwCAQAxggKGMIICggIBADBqMFIxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxIzAhBgNVBAMMGmxvZy10ZXN0LXN0bzMtMS5rb21yZWcubmV0AhRdaHNQ11EJCml8OOzq2F07dXaxqjANBgkqhkiG9w0BAQEFAASCAgC6Z66UDn0xrfaTFJpZFu9+GnO6PToZns+21IcwIaFSXrX/pe7tsXCp5WV+1+McEe57MIln0SYip5mukbtZvHAxMlrOa6YDP4rFYmxQ3AQbOTtomgU0KkH6P8nCQF49pwm7S93nLGcqG0l3h/aj5mhe4D4rWXVN+xbMzaWg2h/dmngCc6SsWkP/iB52GsoEldr0Bq9wsJwkMQZn9eTL7Q3OqhvxQwvQhPdQbOcq/IQTwZDaIZUZyoNfmuUR6uYb3Xvwh6H8+dfQSDFYMjF9WdwNVUI7Y4hT1l3eOpuknJtm8znSOpUrj0Urtx4PLq+ah8eJeKyhSF1iNZkQKH2h48aHPBHz6ndT/pibF89lQVK7WaDuuFDLrkYyh7sAi5EJTaLzjUD7WK1E4VZSs+fT0G4QxCvKBPdAYqRn4ZbHZPeXF+NUcjiMr1Wa+RvBqWDYtz8YJa9ntb3637VvhXBG+HZHhB7rWGa1IqJNZ2y+LQOjnMcdVO1Z92e4RVRqWp1G9o9KTrmBSrQpNY2Tf4/beWc77ieWJ5zrYGAung7EFK9/TkA5qxZVGDvmtLqKiwRB9+Pq/sGqqMTh+ypbdSr9uEmpLqf3SWP/8TZP7zrFROjTiWpZo8s3vuTXainXi4bmaUAmQOjCGrgUKt7/srBbxBqP1iUrrEL6x1XPur/TtrUZUDCBjAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQE7L+6e5g2iCsZcbgQenYN4BgLQKTV3BQVSXj7D1xiB3J5LRDLZFZtMuliUr9SDjLNSXWRy7M2FpoMMvQvaQxKrR4Mb9lZqKbJTtoaUBjLP627MYUrGguMjV7T1nxI4Q/ATZLPV3/s0qdDo0ZuR1pkvjf]