From 93b0e76695d9fb7ae3dfdb022a8ce8abbe8f9a65 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Wed, 29 Apr 2020 17:36:58 +0200 Subject: [PATCH 1/3] telegraf installation in prod log servers --- global/overlay/etc/puppet/cosmos-rules.yaml | 2 ++ .../eid/templates/eidas_logs/eidas_logs.erb | 14 +++----------- .../overlay/etc/hiera/data/local.eyaml | 4 ++++ .../telegraf/telegraf.conf.d/50-json-input.conf | 8 ++++++++ .../overlay/etc/hiera/data/local.eyaml | 4 ++++ .../telegraf/telegraf.conf.d/50-json-input.conf | 8 ++++++++ 6 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 log-1.sveidas.se/overlay/etc/hiera/data/local.eyaml create mode 100644 log-1.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf create mode 100644 log-2.sveidas.se/overlay/etc/hiera/data/local.eyaml create mode 100644 log-2.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf diff --git a/global/overlay/etc/puppet/cosmos-rules.yaml b/global/overlay/etc/puppet/cosmos-rules.yaml index d469bf9b..4f76aaaf 100644 --- a/global/overlay/etc/puppet/cosmos-rules.yaml +++ b/global/overlay/etc/puppet/cosmos-rules.yaml @@ -741,6 +741,8 @@ md-eu1.qa.komreg.net: autoupdate: konsulter: eidas_log: + eid::dockerhost: + eid::telegraf sunet::rsyslog: udp_port: 514 udp_client: 94.176.224.0/24 diff --git a/global/overlay/etc/puppet/modules/eid/templates/eidas_logs/eidas_logs.erb b/global/overlay/etc/puppet/modules/eid/templates/eidas_logs/eidas_logs.erb index d2a1b4ab..56610fc2 100644 --- a/global/overlay/etc/puppet/modules/eid/templates/eidas_logs/eidas_logs.erb +++ b/global/overlay/etc/puppet/modules/eid/templates/eidas_logs/eidas_logs.erb @@ -1,14 +1,6 @@ -/var/log/eidas_fticks.log { - rotate 30 - daily - missingok - notifempty - delaycompress - compress - su root syslog -} - -/var/log/eidas_stats.log { +/var/log/eidas_fticks.log +/var/log/eidas_stats.log +{ rotate 30 daily missingok diff --git a/log-1.sveidas.se/overlay/etc/hiera/data/local.eyaml b/log-1.sveidas.se/overlay/etc/hiera/data/local.eyaml new file mode 100644 index 00000000..ae566d16 --- /dev/null +++ b/log-1.sveidas.se/overlay/etc/hiera/data/local.eyaml @@ -0,0 +1,4 @@ + + +--- +influxdb2_token: ENC[PKCS7,MIIDDwYJKoZIhvcNAQcDoIIDADCCAvwCAQAxggJmMIICYgIBADBKMD0xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxDjAMBgNVBAMMBWxvZy0xAgkA8nnsoONsSSwwDQYJKoZIhvcNAQEBBQAEggIADWJgyp2j5MFBnyi96MD1ikJp6G0Ol8VwWyTOFpjvy9Verd/231VZgHm/f/FYwsQpdg3z4oGGN1uN9ykk6NJJpf9Ls8J7BJ9vZupzDPeK3pEIi4B8+EZSAWtOC2xnZhRySReWblN0ZTiONXYaGPxl+oy9Ytq9SdC9UknyNdQJNctQPqhhsiXi5cfIDU4nsson4e1SEXWYWjN0E+S4ytxbP9XGN3CMp2ZVh6woZjwKP5HaCK8GMcUZqw4dTASqNVehUX/5dlKj0xI80UHTHqPHqCly/itYUHNBRMKArxM5qvRVIdk3gxwkqThcXiLqpOFtwNEWRyHboyXBRswk8gzSDjJ085gm4TteiFfB+URKUtpPTwXztFfP/4v5HnEadbmah3BVPeGz39EKf0y5cv0fhfAt1ggdl6Tp0lvnPNn4BAC5v2GI/ocGYyfWGWgHRtpi/XEY/Mu2X9WNV0RjvgwhTq7k/wduwTkjhHXXMqwMvEjhhtlqwR+EJ4sPwWqIFZg6JVlqAn5ZLIFmnGRrPtFHrxZuw4hlCrkr+OlH/tZ+Qv2gn3/B9fqVTxtg10FUeSt1GZZk2zI/D5JKeOyKoZG3qUlrYAlWH0xLHgjfv1uuqv/0LOC2lv2NCsVqoUPSSsymjEL9TBePJgaDsnu9gKPUsO4/gkbxOib83R9SU8+FpAgwgYwGCSqGSIb3DQEHATAdBglghkgBZQMEASoEEIS5G8ZLDaI5Y6QbeUCVu76AYJKFq2lsrbTM3TYdngzNSIuFIeHPF1f9kF9m4oD1slKLOt0Y1rSP1pqOEPUpchTpUTJPB5zpsT2hRBVfmziltwoOpJou/ztSKqQgYawlra5G05pAeIkaRklgpq/1qAf83w==] diff --git a/log-1.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf b/log-1.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf new file mode 100644 index 00000000..09ab4dcf --- /dev/null +++ b/log-1.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf @@ -0,0 +1,8 @@ +[[inputs.tail]] + files = ["/var/log/eidas_stats.log"] + data_format = "json" + tag_keys = ["requester","country"] + json_name_key = "type" + json_time_key = "timestamp" + json_time_format = "unix_ms" + json_timezone = "Local" diff --git a/log-2.sveidas.se/overlay/etc/hiera/data/local.eyaml b/log-2.sveidas.se/overlay/etc/hiera/data/local.eyaml new file mode 100644 index 00000000..88ca98f0 --- /dev/null +++ b/log-2.sveidas.se/overlay/etc/hiera/data/local.eyaml @@ -0,0 +1,4 @@ + + +--- +influxdb2_token: ENC[PKCS7,MIIDDwYJKoZIhvcNAQcDoIIDADCCAvwCAQAxggJmMIICYgIBADBKMD0xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxDjAMBgNVBAMMBWxvZy0yAgkArNfCT2Xiw0EwDQYJKoZIhvcNAQEBBQAEggIAC9Ywvj4/mZT++Xt1SOGuTG/FtbLXF+IucV4Gm8jVBY/Y4wLwx37FV+DPLfafB/GF0XK7cI6gx0cS7zDpEqfVmWie7/Fklz1J//67msA00MDm6TfIhZo99aRjOFDhEvR8kS8izCifu+O/aRJ4xmHkbXzCvdXoq/VVeFp7vhMd842Z/JJy911qY2E3Hzhf7ua2hXEDbvlSMw3fqgX2jVVXouAaZMFqAUVZCFwMqzRQKrmEw2tigHfyDHzZoYl51bsnAz6Yioi8Kai1Io5YDePZ1FUmQgz5GqlGGcFmGf0t4ls31f5Jjxa08tiSGk1snmX4T+iDhl09t/9lMuu1PQ6gfNO87QUn397RREd3qRVaL301Rh0+tRYcpbBiqCnBvin11tW1wRKFgyVb4HjfOUQdsHnvkjDOv98bYBk936O32lGhnpE291D9ryA7sVAg39b3XRemFWgcHbbkkhOV4Bt4GaMFCOZdGprwCG+MExoYnkn1bHxNyIAS1JSBWW+2Le0SwAiEb49FjNlEIY7/R0U6gJOQksiW3XHNHlvjQI2nE/lOs+1vnMpMYnsCNpQ/K/u5GlYzJasuBGka3GOUBkk0feyVO4RddCbqALEuMNPjQJuQrOy9RcAlwK3P5OlDX+wWpBXA5pRFp3NxV1PnsmV9c7Msuxuxvly0zPAfzJJCjWIwgYwGCSqGSIb3DQEHATAdBglghkgBZQMEASoEEKOaC1XqMZcsCb92Hq8KpfmAYIGcspZnLr+EfVoH2DmGUbgZyaAsVcIIniyep0GHqVKpbHf9+z902Z93aaMiEebpgvjPys0h9GDeyhD0w7FXZhzL221B9p9vlMJrxI3a9ZVfnCeW27Cfy+UuxG1q/yq1OQ==] diff --git a/log-2.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf b/log-2.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf new file mode 100644 index 00000000..09ab4dcf --- /dev/null +++ b/log-2.sveidas.se/overlay/etc/telegraf/telegraf.conf.d/50-json-input.conf @@ -0,0 +1,8 @@ +[[inputs.tail]] + files = ["/var/log/eidas_stats.log"] + data_format = "json" + tag_keys = ["requester","country"] + json_name_key = "type" + json_time_key = "timestamp" + json_time_format = "unix_ms" + json_timezone = "Local" From c3f49aa05bec0c6d08dd2b7f59387b4bca8287f6 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Wed, 29 Apr 2020 17:38:29 +0200 Subject: [PATCH 2/3] syntax misake in cosmos-rules.yaml is fixed --- global/overlay/etc/puppet/cosmos-rules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global/overlay/etc/puppet/cosmos-rules.yaml b/global/overlay/etc/puppet/cosmos-rules.yaml index 4f76aaaf..4d6b448d 100644 --- a/global/overlay/etc/puppet/cosmos-rules.yaml +++ b/global/overlay/etc/puppet/cosmos-rules.yaml @@ -742,7 +742,7 @@ md-eu1.qa.komreg.net: konsulter: eidas_log: eid::dockerhost: - eid::telegraf + eid::telegraf: sunet::rsyslog: udp_port: 514 udp_client: 94.176.224.0/24 From 5f26c773f7706be1d8b592a1fb97c9588a41e7c6 Mon Sep 17 00:00:00 2001 From: Maria Haider Date: Wed, 29 Apr 2020 17:38:34 +0200 Subject: [PATCH 3/3] update db --- global/overlay/etc/puppet/cosmos-db.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/global/overlay/etc/puppet/cosmos-db.yaml b/global/overlay/etc/puppet/cosmos-db.yaml index 63675969..a8cc860a 100644 --- a/global/overlay/etc/puppet/cosmos-db.yaml +++ b/global/overlay/etc/puppet/cosmos-db.yaml @@ -835,6 +835,8 @@ classes: log-1.sveidas.se: autoupdate: null common: null + eid::dockerhost: null + eid::telegraf: null eidas_log: null entropyclient: null infra_ca_rp: null @@ -847,6 +849,8 @@ classes: log-2.sveidas.se: autoupdate: null common: null + eid::dockerhost: null + eid::telegraf: null eidas_log: null entropyclient: null infra_ca_rp: null @@ -1232,13 +1236,13 @@ members: eidas-redis-3.sveidas.se, eidas-redis-4.sveidas.se, eidas-redis-fe-1.sveidas.se, eidas-redis-fe-2.sveidas.se, eidas-test-1.sveidas.se, eidas-test-2.sveidas.se, eumd-1.komreg.net, eumd-2.komreg.net, fe-fre-3.komreg.net, fe-tug-3.komreg.net, - log.qa.sveidas.se, monitor-tug-3.komreg.net, natmd-1.komreg.net, natmd-2.komreg.net, - nic.komreg.net, prid-1.sveidas.se, prid-2.sveidas.se, refidp-1.qa.sveidas.se, - test-1.qa.sveidas.se] + log-1.sveidas.se, log-2.sveidas.se, log.qa.sveidas.se, monitor-tug-3.komreg.net, + natmd-1.komreg.net, natmd-2.komreg.net, nic.komreg.net, prid-1.sveidas.se, prid-2.sveidas.se, + refidp-1.qa.sveidas.se, test-1.qa.sveidas.se] eid::kvmhost: [kvmdemw-fre-3a.komreg.net, kvmdemw-fre-3b.komreg.net, kvmeidas-fre-3.komreg.net, kvmeidas-tug-3.komreg.net, kvmfe-fre-3.komreg.net, kvmfe-tug-3.komreg.net, kvminfra-fre-3.komreg.net, kvminfra-tug-3.komreg.net, kvmmeta-fre-3.komreg.net, kvmmeta-tug-3.komreg.net] - eid::telegraf: [log.qa.sveidas.se] + eid::telegraf: [log-1.sveidas.se, log-2.sveidas.se, log.qa.sveidas.se] eidas_connector: [eidas-connector-1.sveidas.se, eidas-connector-2.sveidas.se, eidas-connector-3.sveidas.se, eidas-connector-4.sveidas.se, eidas-node-1.qa.sveidas.se] eidas_de_middleware: [demw-1.qa.sveidas.se]