From cbe64bca5e16eab3f50a4ecbcf927d250c70f932 Mon Sep 17 00:00:00 2001 From: Rasmus Thorslund Date: Mon, 30 Sep 2024 14:54:58 +0200 Subject: [PATCH 1/2] lb-tug-test-1.sunet.se added --- lb-tug-test-1.sunet.se/README | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 lb-tug-test-1.sunet.se/README diff --git a/lb-tug-test-1.sunet.se/README b/lb-tug-test-1.sunet.se/README new file mode 100644 index 0000000..a18dac1 --- /dev/null +++ b/lb-tug-test-1.sunet.se/README @@ -0,0 +1,3 @@ + +The system documentation is in the docs directory of the multiverse repository. + From 9fc0bb3539caa76dfce20b4005d3902138ccacb7 Mon Sep 17 00:00:00 2001 From: Rasmus Thorslund Date: Mon, 30 Sep 2024 14:58:22 +0200 Subject: [PATCH 2/2] added renamed lb-tug-test-1 --- .../overlay/etc/hiera/data/group.yaml | 60 +++++++++ .../health-checks.d/lb_healthcheck.py.check | 71 +++++++++++ .../frontend/config/common/haproxy_base.j2 | 116 ++++++++++++++++++ .../opt/frontend/config/common/haproxy_idp.j2 | 24 ++++ .../frontend/config/common/haproxy_macros.j2 | 77 ++++++++++++ .../frontend/config/edusealapit/haproxy.j2 | 20 +++ .../frontend/config/thissstaging/haproxy.j2 | 1 + 7 files changed, 369 insertions(+) create mode 100644 lb-test-common/overlay/etc/hiera/data/group.yaml create mode 100755 lb-test-common/overlay/etc/sunet-machine-healthy/health-checks.d/lb_healthcheck.py.check create mode 100644 lb-test-common/overlay/opt/frontend/config/common/haproxy_base.j2 create mode 100644 lb-test-common/overlay/opt/frontend/config/common/haproxy_idp.j2 create mode 100644 lb-test-common/overlay/opt/frontend/config/common/haproxy_macros.j2 create mode 100644 lb-test-common/overlay/opt/frontend/config/edusealapit/haproxy.j2 create mode 100644 lb-test-common/overlay/opt/frontend/config/thissstaging/haproxy.j2 diff --git a/lb-test-common/overlay/etc/hiera/data/group.yaml b/lb-test-common/overlay/etc/hiera/data/group.yaml new file mode 100644 index 0000000..94383c3 --- /dev/null +++ b/lb-test-common/overlay/etc/hiera/data/group.yaml @@ -0,0 +1,60 @@ +--- +sunet_frontend: + + load_balancer: + haproxy_imagetag: '20230228-stable' + api_imagetag: 'stable' + exabgp_imagetag: 'stable' + + peers: + se-tug-rs-2.sunet.se: + as: '65434' + remote_ip: '192.36.171.71' + se-tug-rs-2.sunet.se_v6: + as: '65434' + remote_ip: '2001:6b0:8:7::71' + se-sthb-rs-1.sunet.se: + as: '65434' + remote_ip: '192.36.171.130' + se-sthb-rs-1.sunet.se_v6: + as: '65434' + remote_ip: '2001:6b0:8:1::130' + + websites: + 'thissstaging': + site_name: 'md-staging.thiss.io' + frontends: + 'tug-lb-1.sunet.se': + ips: ['37.156.192.5', '2001:6b0:60:c0::5'] + 'sthb-lb-1.sunet.se': + ips: ['37.156.192.6', '2001:6b0:60:c0::6'] + backends: + default: + 'md-staging-1.thiss.io': + ips: ['89.47.184.98'] + server_args: 'check verify none' + allow_ports: + - 80 + - 443 + letsencrypt_server: 'acme-c.sunet.se' + haproxy_imagetag: '20230228-stable' + frontendtools_imagetag: '20230228' + +# 'edusealapit': +# site_name: 'test-api.eduseal.sunet.se' +# frontends: +# 'tug-lb-1.sunet.se': +# ips: ['37.156.192.18', '2001:6b0:60:c0::18'] +# 'sthb-lb-1.sunet.se': +# ips: ['37.156.192.19', '2001:6b0:60:c0::19'] +# backends: +# default: +# 'car-test-1.eduseal.sunet.se': +# ips: ['89.45.237.159'] +# server_args: 'ssl check verify none' +# allow_ports: +# - 80 +# - 443 +# letsencrypt_server: 'acme-c.sunet.se' +# haproxy_imagetag: '20230228-stable' +# frontendtools_imagetag: '20230228' diff --git a/lb-test-common/overlay/etc/sunet-machine-healthy/health-checks.d/lb_healthcheck.py.check b/lb-test-common/overlay/etc/sunet-machine-healthy/health-checks.d/lb_healthcheck.py.check new file mode 100755 index 0000000..4c95aab --- /dev/null +++ b/lb-test-common/overlay/etc/sunet-machine-healthy/health-checks.d/lb_healthcheck.py.check @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import yaml +import subprocess +import time +import sys + +groupyaml = '/etc/hiera/data/group.yaml' + +def get_frontends(data): + try: + return list(data['sunet_frontend']['load_balancer']['websites'].keys()) + except KeyError: + return [] + +def check_docker_instance_status(instance): + cmd = f"docker inspect -f {r'{{.State.Status}}'} {instance}" + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + return result.stdout.strip() == 'running' + +def is_exabgp_running(): + cmd = ["systemctl", "is-active", "exabgp.service"] + result = subprocess.run(cmd, capture_output=True, text=True) + return result.stdout.strip() == 'active' + +def check_docker_instances(instances, max_retries=3, initial_wait=10): + for instance in instances: + retries = 0 + while retries < max_retries: + if check_docker_instance_status(instance): + print(f"Instance: {instance} is running!") + break + else: + print(f"Instance: {instance} is not running! Will try again in {initial_wait * (2**retries)} seconds.") + time.sleep(initial_wait * (2**retries)) + retries += 1 + if retries == max_retries: + print(f"Max retries reached for instance: {instance}, exiting!") + sys.exit(1) + +def check_exabgp_running(max_retries=3, initial_wait=10): + retries = 0 + while retries < max_retries: + if is_exabgp_running(): + print("ExaBGP service is running!") + break + else: + print(f"Exabgp is not running! Will try again in {initial_wait * (2**retries)} seconds.") + time.sleep(initial_wait * (2**retries)) + retries += 1 + if retries == max_retries: + print(f"Max retries reached for checking if exabgp is running, exiting!") + sys.exit(1) + +with open(groupyaml, 'r') as f: + data = yaml.safe_load(f) + +frontends = get_frontends(data) +instances = [] + +for frontend in frontends: + instances.append(frontend + '-haproxy-1') + instances.append(frontend + '-monitor-1') + instances.append(frontend + '-config-1') + +instances.append('frontend-api-1') +instances.append('frontend-telegraf-1') + +check_exabgp_running() +check_docker_instances(instances) +sys.exit(0) diff --git a/lb-test-common/overlay/opt/frontend/config/common/haproxy_base.j2 b/lb-test-common/overlay/opt/frontend/config/common/haproxy_base.j2 new file mode 100644 index 0000000..2a763aa --- /dev/null +++ b/lb-test-common/overlay/opt/frontend/config/common/haproxy_base.j2 @@ -0,0 +1,116 @@ +# haproxy for SUNET frontend load balancer nodes. +# +{% from "common/haproxy_macros.j2" import output_backends %} + +{% block global %} +global + log stdout format raw local0 debug + + daemon + maxconn 256 + stats socket /haproxy_control/stats mode 660 + #server-state-file /tmp/server_state + + # whole container is started as non-root + #user haproxy + #group haproxy + + # Default SSL material locations + ca-base /etc/ssl/certs + crt-base /etc/ssl/private + + # Mozilla Guideline v5.7 intermediate configuration + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305 + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets + + ssl-default-server-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305 + ssl-default-server-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets + # end Mozilla config + + tune.ssl.default-dh-param 2048 + + spread-checks 20 + + + +{% endblock global %} + + +{% block defaults %} +defaults + log global + mode http + option httplog + option dontlognull + option redispatch + option forwardfor + # funny looking values because recommendation is to have these slightly + # above mulitples of three seconds to play nice with TCP resend timers + timeout check 5s + timeout connect 4s + timeout client 17s + timeout server 17s + timeout http-request 5s + + # never fail on address resolution + default-server init-addr libc,none + balance roundrobin +{% endblock defaults %} + +{% block stats %} +frontend LB-http + # expose stats info over HTTP to exabgp + bind 127.0.0.1:9000 + http-request set-log-level silent + default_backend LB + +backend LB + stats enable + #stats hide-version + stats uri /haproxy_stats +{% endblock stats %} + + +{% block global_backends %} +{% if letsencrypt_server is defined %} +backend letsencrypt_{{ letsencrypt_server }} + server letsencrypt_{{ letsencrypt_server }} {{ letsencrypt_server }}:80 +{% else %} +# letsencrypt_backend not defined +{% endif %} +{% endblock global_backends %} + + +{% block https_everything %} +# +# Redirect _everything_ to HTTPS +frontend http-frontend + bind 0.0.0.0:80 + bind :::80 + + redirect scheme https code 301 if !{ ssl_fc } ! { path_beg /.well-known/acme-challenge/ } +{% if letsencrypt_server is defined %} + use_backend letsencrypt_{{ letsencrypt_server }} if { path_beg /.well-known/acme-challenge/ } +{% else %} + # letsencrypt_backend not defined +{% endif %} +{% endblock https_everything %} + +# +# Frontend section +# +{% block frontend %} +{% endblock frontend %} + + +# +# Backend section +# +{% block pre_backend %} +{% endblock pre_backend %} + +{% block backend %} +{{ output_backends(backends, config=[]) }} +{% endblock backend %} diff --git a/lb-test-common/overlay/opt/frontend/config/common/haproxy_idp.j2 b/lb-test-common/overlay/opt/frontend/config/common/haproxy_idp.j2 new file mode 100644 index 0000000..633a67e --- /dev/null +++ b/lb-test-common/overlay/opt/frontend/config/common/haproxy_idp.j2 @@ -0,0 +1,24 @@ +{% extends 'common/haproxy_base.j2' %} + +{% from "common/haproxy_macros.j2" import bind_ip_tls, web_security_options, acme_challenge, csp %} + +{% block frontend %} +frontend {{ site_name }} + {{ bind_ip_tls(bind_ips, 443, tls_certificate_bundle) }} + + + timeout http-request 10s + timeout http-keep-alive 4s + option forwardfor + http-request set-header X-Forwarded-Proto https + + {{ web_security_options(['no_frames', 'block_xss', 'hsts', 'no_sniff', 'no_cache']) }} + + {{ csp(["default-src " + [csp_app_src]|join(' '), + "script-src " + [csp_script_src]|join(' '), + ]) }} + + {{ acme_challenge(letsencrypt_server) }} + + use_backend {{ site_name }}__default +{% endblock frontend %} diff --git a/lb-test-common/overlay/opt/frontend/config/common/haproxy_macros.j2 b/lb-test-common/overlay/opt/frontend/config/common/haproxy_macros.j2 new file mode 100644 index 0000000..dec367c --- /dev/null +++ b/lb-test-common/overlay/opt/frontend/config/common/haproxy_macros.j2 @@ -0,0 +1,77 @@ +# +# Macros +# + +{%- macro bind_ip_tls(bind_ips, port, tls_cert) -%} +{%- for ip in bind_ips %} + bind {{ ip }}:{{ port }} ssl crt {{ tls_cert }} +{%- endfor %} +{%- endmacro %} + + +{%- macro web_security_options(list) -%} +{%- for this in list %} +{%- if this == 'no_frames' %} + # Do not allow rendering the site within an frame, which prevents clickjacking. + http-response set-header X-Frame-Options "DENY" + +{% endif %} +{%- if this == 'block_xss' %} + # Enable browser supplied XSS-protection, even if has been turned off. + # If XSS is detected by the browser, block it instead of trying to sanitize it. + http-response set-header X-XSS-Protection "1; mode=block" + +{% endif %} +{%- if this == 'hsts' %} + # 20 years in seconds is 630720000 (86400 * 365 * 20) + http-response set-header Strict-Transport-Security "max-age=630720000" + +{% endif %} +{%- if this == 'no_sniff' %} + # Prevent MIME-confusion attacks that can lead to e.g. XSS + http-response set-header X-Content-Type-Options "nosniff" + +{% endif %} +{%- if this == 'no_cache' %} + # The information is intended for a single user and must not + # be cached by a shared cache and should always be revalidated. + http-response set-header Cache-Control "no-cache, no-store, must-revalidate" + http-response set-header Pragma "no-cache" + http-response set-header Expires "0" + +{% endif %} +{%- endfor %} +{%- endmacro %} + + +{%- macro acme_challenge(letsencrypt_server) -%} +{%- if letsencrypt_server is defined %} + use_backend letsencrypt_{{ letsencrypt_server }} if { path_beg /.well-known/acme-challenge/ } +{%- else %} + # No letsencrypt_server specified +{%- endif %} +{%- endmacro %} + +{%- macro csp(data) -%} + # Content Security Policy + http-response set-header Content-Security-Policy "{{ data|join('; ') }}" +{%- endmacro %} + +{%- macro output_backends(backends, config=[], server_args='') -%} +{% if backends is defined %} +{%- for this in backends %} +backend {{ this.name }} + {{ config|join('\n ') }} + {%- for server in this.servers %} + {%- if server.server_args is defined %} + {%- set server_args = server.server_args %} + {%- endif %} + {% if server is defined %} + server {{ server.server }}_{{ server.address_family }} {{ server.ip }}:{{ server.port }} {{ server_args }} + {%- endif %} + {%- endfor %} +{%- endfor %} +{% else %} +# No backends found in context +{% endif %} +{%- endmacro %} diff --git a/lb-test-common/overlay/opt/frontend/config/edusealapit/haproxy.j2 b/lb-test-common/overlay/opt/frontend/config/edusealapit/haproxy.j2 new file mode 100644 index 0000000..c8369d1 --- /dev/null +++ b/lb-test-common/overlay/opt/frontend/config/edusealapit/haproxy.j2 @@ -0,0 +1,20 @@ +{% extends 'common/haproxy_base.j2' %} + +{% from "common/haproxy_macros.j2" import bind_ip_tls, web_security_options, acme_challenge, csp %} + +{% block frontend %} +frontend {{ site_name }} + {{ bind_ip_tls(bind_ips, 443, tls_certificate_bundle) }} + + + timeout http-request 10s + timeout http-keep-alive 4s + option forwardfor + http-request set-header X-Forwarded-Proto https + + {{ web_security_options(['no_cache', 'block_xss', 'hsts', 'no_sniff']) }} + + {{ acme_challenge(letsencrypt_server) }} + + use_backend {{ site_name }}__default +{% endblock frontend %} diff --git a/lb-test-common/overlay/opt/frontend/config/thissstaging/haproxy.j2 b/lb-test-common/overlay/opt/frontend/config/thissstaging/haproxy.j2 new file mode 100644 index 0000000..fff9be9 --- /dev/null +++ b/lb-test-common/overlay/opt/frontend/config/thissstaging/haproxy.j2 @@ -0,0 +1 @@ +{% extends 'common/haproxy_idp.j2' %}