Use correct output filename

Also teach sunet-l4lb-namespace to load the nft ruleset if it exists.
While here modify the script so instead of running "once per netns
config file" we merge the interface config from each json file into the
same dict per namespace. Without this we would attempt to load the nft
ruleset twice (once per file that mentioned the namespace) or warn twice
if the file did not exist etc.

edit-secrets

@@ -67,6 +67,39 @@ function patch_broken_eyaml {
                next if @@plugins.include? spec
 
                dependency = spec.dependencies.find { |d| d.name == "hiera-eyaml" }
+EOF
+            fi
+        fi
+    fi
+
+    #
+    # Ubuntu 24.04 (noble) has a hiera-eyaml version that is incompatible with ruby 3.2+ (default in ubuntu24).
+    # This is fixed in hiera-eyaml version 3.3.0: https://github.com/voxpupuli/hiera-eyaml/pull/340/files
+    # https://github.com/voxpupuli/hiera-eyaml/blob/master/CHANGELOG.md
+    # But there is no modern version of hiera-eyaml packaged in debian or ubuntu.
+    # https://github.com/puppetlabs/puppet/wiki/Puppet-8-Compatibility#filedirexists-removed
+    #
+
+    . /etc/os-release
+    if [ "${VERSION_CODENAME}" == "noble" ]; then
+        plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb"
+        if [ -f $plugins_file ]; then
+            # We only want to try patching the file if it is the known broken version
+            bad_sum="59c6eb910ab2eb44f8c75aeaa79bff097038feb673b5c6bdccde23d9b2a393e2"
+            sum=$(sha256sum $plugins_file | awk '{print $1}')
+            if [ "$sum" == "$bad_sum" ]; then
+                patch --fuzz=0 --directory=/ --strip=0 <<'EOF'
+--- /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb.orig	2022-06-11 16:30:10.000000000 +0000
++++ /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb	2024-09-09 14:13:19.306342025 +0000
+@@ -59,7 +59,7 @@
+             Optimist::die "You must specify an eyaml file" if ARGV.empty?
+             options[:source] = :eyaml
+             options[:eyaml] = ARGV.shift
+-            if File.exists? options[:eyaml]
++            if File.exist? options[:eyaml]
+               begin
+                 options[:input_data] = File.read options[:eyaml]
+               rescue
 EOF
             fi
         fi

global/overlay/etc/puppet/cosmos-rules.yaml

@@ -6,7 +6,7 @@
     disable_ipv6_privacy: true
     disable_all_local_users: true
     install_scriptherder: true
-    ntpd_config: false
+    ntpd_config: true
 
 '^cdn-prod-l4lb-[0-9]+.sunet.se$':
   sunet::dockerhost2:
@@ -19,10 +19,11 @@
 '^internal-.+-test-mqtt-[0-9]+\.cdn\.sunet\.se$':
   cdn::ca_trust:
   cdn::mqtt:
-    dc: tug
+    dc: sto3
     clients:
-      - shared-tug-test-cache-1.cdn.sunet.se
-      - shared-tug-test-cache-2.cdn.sunet.se
+      - internal-sto3-test-cache-2.cdn.sunet.se
+    client_ips:
+      - 192.36.171.94 # internal-sto3-test-cache-2.cdn.sunet.se
 
 '^internal-.+-test-ca-[0-9]+\.cdn\.sunet\.se$':
   sunet::dockerhost2:
@@ -31,3 +32,12 @@
 '^internal-.+-test-cache-[0-9]+\.cdn\.sunet\.se$':
   sunet::dockerhost2:
   cdn::cache:
+  sunet::certbot::sync::client:
+
+'^internal-.+-test-cs-[0-9]+\.cdn\.sunet\.se$':
+  sunet::certbot::acmed:
+  sunet::certbot::sync::server:
+
+'^internal-.+-test-db-[0-9]+\.cdn\.sunet\.se$':
+  sunet::dockerhost2:
+  cdn::db:

global/overlay/etc/puppet/modules/cdn/files/ca/configure-acme

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# When initializing step-ca with the docker flag DOCKER_STEPCA_INIT_ACME
+# a basic ACME provisioner is enabled. This script runs commands to modify the
+# default configuration.
+
+# Enable forceCN if not set.
+# This is needed because certbot does not include a
+# Subject CN field in the CSR:
+# https://github.com/certbot/certbot/issues/9633#issuecomment-1484988078
+# ... and the Mosquitto MQTT server uses the Subject CN in ACL filters.
+#
+# Ideally Mosquitto would learn to look at the SAN field instead:
+# https://github.com/eclipse-mosquitto/mosquitto/issues/2511
+if [ "$(step ca provisioner list | jq -r '.[] | select (.name == "acme") | .forceCN')" = "null" ]; then
+    step ca provisioner update acme --force-cn --admin-subject=step --admin-provisioner=admin  --admin-password-file=/opt/step-ca/init/secrets/provisioner-password
+fi

global/overlay/etc/puppet/modules/cdn/files/ca/set-provisioner-pw

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-# When initializing step-ca with the docker flag STEPCA_INIT_PASSWORD_FILE the
-# password will be used both for key encryption as well as the admin "step"
-# provisioner. If not using that flag a separate password will be generated for
-# each usage. This seems better as you are not typing the encryption password
-# any other time, while the provisioner password is used anytime you are
-# managing things.
+# When initializing step-ca with the docker flag
+# DOCKER_STEPCA_INIT_PASSWORD_FILE the password will be used both for key
+# encryption as well as the admin "step" provisioner. If not using that flag a
+# separate password will be generated for each usage. This seems better as you
+# are not typing the encryption password any other time, while the provisioner
+# password is used anytime you are managing things.
 #
 # This script is used on first setup of step-ca to modify the provisioner to
 # use its own password instead. Pending

global/overlay/etc/puppet/modules/cdn/files/db/init-cdn-db.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+# shellcheck source=/dev/null
+. /conf/init-cdn-db.conf
+
+# Create database named after user, then create a schema named the same as the
+# user which is also owned by that user. Because search_path (SHOW
+# search_path;) starts with "$user" by default this means any tables will be
+# created in that user-specific SCHEMA by default instead of falling back to
+# "public". This follows the "secure schema usage pattern" summarized as
+# "Constrain ordinary users to user-private schemas" from
+# https://www.postgresql.org/docs/current/ddl-schemas.html#DDL-SCHEMAS-PATTERNS
+#
+# "In PostgreSQL 15 and later, the default configuration supports this usage
+# pattern. In prior versions, or when using a database that has been upgraded
+# from a prior version, you will need to remove the public CREATE privilege
+# from the public schema"
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+	CREATE USER cdn WITH PASSWORD '${cdn_password:?}';
+	CREATE DATABASE cdn;
+	GRANT ALL PRIVILEGES ON DATABASE cdn TO cdn;
+	\c cdn;
+	CREATE SCHEMA cdn AUTHORIZATION cdn;
+EOSQL

global/overlay/etc/puppet/modules/cdn/files/l4lb/sunet-l4lb-namespace

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+# pylint:disable=invalid-name
+# pylint:enable=invalid-name
+# pylint:disable=invalid-name
+# pylint:enable=invalid-name
+"""
+netplan does not have network namespace support so configure by hand
+
+Tools used before committing code:
+black sunet-l4lb-namespace
+isort sunet-l4lb-namespace
+pylint sunet-l4lb-namespace
+mypy --strict sunet-l4lb-namespace
+"""
+
+import ipaddress
+import json
+import os
+import shlex
+import subprocess
+import sys
+
+
+def run_command(cmd: str) -> subprocess.CompletedProcess[str]:
+    """Execute subprocess command"""
+    print(f"{cmd}")
+    args = shlex.split(cmd)
+    try:
+        proc = subprocess.run(args, capture_output=True, check=True, encoding="utf-8")
+    except subprocess.CalledProcessError as exc:
+        stderr_str = exc.stderr.rstrip()
+        print(
+            f"command failed: cmd='{cmd}', rc={exc.returncode}, stderr='{stderr_str}'",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    return proc
+
+
+def configure_interfaces(  # pylint: disable=too-many-locals,too-many-branches
+    namespace: str, if_data: dict[str, dict[str, list[str]]]
+) -> None:
+    """Configure interfaces"""
+    proc = run_command(f"ip netns exec {namespace} ip -j addr show")
+    namespace_ifs = json.loads(proc.stdout)
+
+    ipv4_key = "ipv4"
+    ipv6_key = "ipv6"
+    for if_name, data in if_data.items():
+        if_exists = next(
+            (True for interface in namespace_ifs if interface["ifname"] == if_name),
+            False,
+        )
+        if not if_exists:
+            if if_name.startswith("dummy"):
+                run_command(
+                    f"ip netns exec {namespace} ip link add {if_name} type dummy"
+                )
+            else:
+                run_command(f"ip link set {if_name} netns {namespace}")
+
+            run_command(f"ip netns exec {namespace} ip link set {if_name} up")
+
+        proc = run_command(f"ip netns exec {namespace} ip -j addr show dev {if_name}")
+        if_conf = json.loads(proc.stdout)
+
+        # Add missing addresses from config
+        if ipv4_key in data:
+            for configured_ipv4_cidr in data[ipv4_key]:
+                ip4, prefix = configured_ipv4_cidr.split("/")
+                v4_addr_exists = next(
+                    (
+                        True
+                        for addr in if_conf[0]["addr_info"]
+                        if addr["local"] == ip4 and addr["prefixlen"] == int(prefix)
+                    ),
+                    False,
+                )
+                if not v4_addr_exists:
+                    run_command(
+                        f"ip netns exec {namespace} ip addr add {configured_ipv4_cidr} dev {if_name}"  # pylint: disable=line-too-long
+                    )
+        if ipv6_key in data:
+            for ipv6_cidr in data[ipv6_key]:
+                ip6, prefix = ipv6_cidr.split("/")
+                v6_addr_exists = next(
+                    (
+                        True
+                        for addr in if_conf[0]["addr_info"]
+                        if addr["local"] == ip6 and addr["prefixlen"] == int(prefix)
+                    ),
+                    False,
+                )
+                if not v6_addr_exists:
+                    run_command(
+                        f"ip netns exec {namespace} ip addr add {ipv6_cidr} dev {if_name}"
+                    )
+
+        # Remove no longer configured addresseses
+        for addr_info in if_conf[0]["addr_info"]:
+            # Ignore addresses like fe80
+            if addr_info["scope"] != "global":
+                continue
+
+            cidr = "/".join((addr_info["local"], str(addr_info["prefixlen"])))
+
+            # We need strict=False because otherwise ip_network() gets angry if
+            # there are host bits set in the address (which of course there is
+            # because we are parsing actual interface configs, not pure
+            # "networks")
+            cidr_net = ipaddress.ip_network(cidr, strict=False)
+
+            needs_removal = False
+            if cidr_net.version == 4:
+                if ipv4_key not in data or cidr not in data[ipv4_key]:
+                    needs_removal = True
+            elif cidr_net.version == 6:
+                if ipv6_key not in data or cidr not in data[ipv6_key]:
+                    needs_removal = True
+            else:
+                raise ValueError(
+                    f"Expected IPv4 or IPv6, got something else: {cidr_net.version}"
+                )
+
+            if needs_removal:
+                run_command(
+                    f"ip netns exec {namespace} ip addr del {cidr} dev {if_name}"
+                )
+
+
+def setup_namespaces(netns_data: dict[str, dict[str, dict[str, list[str]]]]) -> None:
+    """Setup network namespaces"""
+    proc = run_command("ip -j netns list")
+    existing_netns = json.loads(proc.stdout)
+    for namespace, if_data in netns_data.items():
+        netns_exists = next(
+            (True for netns in existing_netns if netns["name"] == namespace), False
+        )
+        if not netns_exists:
+            run_command(f"ip netns add {namespace}")
+
+            # Make localhost available
+            run_command(f"ip netns exec {namespace} ip link set lo up")
+
+        # (Re)load the nft ruleset for the given namespace
+        nft_ruleset = f"/opt/sunet-cdn/l4lb/conf/nft-{namespace}.conf"
+        if os.path.isfile(nft_ruleset):
+            run_command(f"ip netns exec {namespace} nft -f {nft_ruleset}")
+        else:
+            print(
+                f"WARNING: no nft ruleset found for namespace '{namespace}' ({nft_ruleset}), the namespace will not be firewalled"  # pylint: disable=line-too-long
+            )
+
+        configure_interfaces(namespace, if_data)
+
+
+def main() -> None:
+    """Starting point of the program"""
+
+    # JSON file format:
+    # {
+    #  "namespace1": {
+    #    "interface1": {
+    #      "ipv4": [
+    #        "192.168.10.1/31"
+    #      ],
+    #      "ipv6": [
+    #        "2001:db8:1337:74::1/127"
+    #      ]
+    #    },
+    #    "interface2": {
+    #      "ipv4": [
+    #        "192.168.10.3/31"
+    #      ],
+    #      "ipv6": [
+    #        "2001:db8:1338:75::1/127"
+    #      ]
+    #    }
+    #  }
+    # }
+
+    input_files = [
+        "/opt/sunet-cdn/l4lb/conf/netns-base.json",
+        "/opt/sunet-cdn/l4lb/conf/netns-sunet-cdn-agent.json",
+    ]
+
+    merged_netns_data: dict[str, dict[str, dict[str, list[str]]]] = {}
+    for input_file in input_files:
+        try:
+            with open(input_file, encoding="utf-8") as f:
+                netns_data = json.load(f)
+
+            # Combine interface config from multiple files belonging to the same namespace
+            for ns, ns_data in netns_data.items():
+                if ns in merged_netns_data:
+                    merged_netns_data[ns].update(ns_data)
+                else:
+                    merged_netns_data[ns] = ns_data
+
+        except FileNotFoundError:
+            print(f"skipping nonexistant file '{input_file}'")
+            continue
+
+    setup_namespaces(merged_netns_data)
+
+
+if __name__ == "__main__":
+    main()

global/overlay/etc/puppet/modules/cdn/files/mqtt/sunet-cdn-mqtt

@@ -9,20 +9,13 @@ set -eu
 le_dir="/etc/letsencrypt/live/$(hostname -f)"
 mosquitto_dir="/etc/mosquitto"
 
-le_chain="$le_dir/chain.pem"
-mosquitto_chain="$mosquitto_dir/ca_certificates/chain.pem"
-cp "$le_chain" "$mosquitto_chain"
-chown mosquitto:root "$mosquitto_chain"
-
-le_cert="$le_dir/cert.pem"
-mosquitto_cert="$mosquitto_dir/certs/cert.pem"
-cp "$le_cert" "$mosquitto_cert"
-chown mosquitto:root "$mosquitto_cert"
+le_fullchain="$le_dir/fullchain.pem"
+mosquitto_fullchain="$mosquitto_dir/certs/fullchain.pem"
+install -m 644 -o mosquitto -g root "$le_fullchain" "$mosquitto_fullchain"
 
 le_key="$le_dir/privkey.pem"
 mosquitto_key="$mosquitto_dir/certs/privkey.pem"
-cp "$le_key" "$mosquitto_key"
-chown mosquitto:root "$mosquitto_key"
+install -m 600 -o mosquitto -g root "$le_key" "$mosquitto_key"
 
 # Tell mosquitto to reload certs
 pkill -x -HUP mosquitto

global/overlay/etc/puppet/modules/cdn/manifests/ca.pp

@@ -59,6 +59,14 @@ class cdn::ca(
     content => file('cdn/ca/bootstrap-client'),
   }
 
+  file { '/opt/step-ca/init/scripts/configure-acme':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => file('cdn/ca/configure-acme'),
+  }
+
   file { '/opt/step-ca/init/deb':
     ensure => directory,
     owner  => 'root',

global/overlay/etc/puppet/modules/cdn/manifests/cache.pp

@@ -2,9 +2,19 @@
 class cdn::cache(
   Hash[String, Integer] $customers = {
     customer1 => 1000000000,
-  }
+  },
+  String $sunet_cdnp_version = '0.0.7',
+  Hash[String, String] $acme_url = {
+    test => 'https://internal-sto3-test-ca-1.cdn.sunet.se:9000/acme/acme/directory'
+  },
+  Hash[String, Hash[String, String]] $mqtt_url = {
+    sto3 => {
+      test => 'tls://internal-sto3-test-mqtt-1.cdn.sunet.se:8883',
+    },
+  },
 )
 {
+  include sunet::systemd_reload
   include sunet::packages::certbot
   include cdn::ca_trust
 
@@ -31,7 +41,7 @@ class cdn::cache(
     mode   => '0755',
   }
 
-  file { "/opt/sunet-cdn/conf/varnish-slash-seccomp.json":
+  file { '/opt/sunet-cdn/conf/varnish-slash-seccomp.json':
     ensure  => file,
     owner   => 'root',
     group   => 'root',
@@ -39,31 +49,221 @@ class cdn::cache(
     content => template('cdn/cache/varnish-slash-seccomp.json.erb'),
   }
 
+  file { '/etc/systemd/network/10-cdn-dummy.netdev':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-dummy.netdev.erb'),
+  }
+
+  file { '/etc/systemd/network/10-cdn-dummy.network':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-dummy.network.erb'),
+  }
+
+  file { '/etc/systemd/network/10-cdn-ipip.netdev':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-ipip.netdev.erb'),
+  }
+
+  file { '/etc/systemd/network/10-cdn-ipip.network':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-ipip.network.erb'),
+  }
+
+  file { '/etc/systemd/network/10-cdn-ip6tunl.netdev.erb':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-ip6tunl.netdev.erb'),
+  }
+
+  file { '/etc/systemd/network/10-cdn-ip6tunl.network.erb':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-ip6tunl.network.erb'),
+  }
+
+  # Reload the network config if it has changed
+  exec { 'networkctl reload':
+    subscribe   => [File['/etc/systemd/network/10-cdn-dummy.network'], File['/etc/systemd/network/10-cdn-ipip.network']],
+    refreshonly => true,
+  }
+
+  $sysctl_file = '/etc/sysctl.d/99-cdn-cache.conf'
+  file { $sysctl_file:
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/sysctl.erb'),
+  }
+  # Load the sysctl file if it has changed
+  exec { "sysctl -p ${sysctl_file}":
+    subscribe   => File[$sysctl_file],
+    refreshonly => true,
+  }
+
+  # Allow IPv4 tunnel packets arriving from l4lb nodes
+  sunet::nftables::rule { 'sunet_cdn_tunnel4':
+    rule => 'add rule inet filter input ip saddr { 130.242.64.233, 130.242.64.235 } ip protocol ipencap counter accept comment "sunet-cdn-tunnel4"'
+  }
+
+  # Allow IPv6 tunnel packets arriving from l4lb nodes
+  sunet::nftables::rule { 'sunet_cdn_tunnel6':
+    rule => 'add rule inet filter input ip6 saddr { 2001:6b0:2006:74::1, 2001:6b0:2006:75::1 } ip6 nexthdr ipv6 counter accept comment "sunet-cdn-tunnel6"'
+  }
+
+  # Allow decapsulated tunnel packets targeting the service IP range to reach
+  # local service ports
+  sunet::nftables::rule { 'sunet_cdn_service4':
+    rule => 'add rule inet filter input meta iifname tunl0 ip daddr 188.240.152.0/24 tcp dport { 80, 443 } counter accept comment "sunet-cdn-service4"'
+  }
+  sunet::nftables::rule { 'sunet_cdn_service6':
+    rule => 'add rule inet filter input meta iifname ip6tnl0 ip6 daddr 2001:6b0:2100::/48 tcp dport { 80, 443 } counter accept comment "sunet-cdn-service6"'
+  }
+
+  # From https://wiki.sunet.se/display/sunetops/Platform+naming+standards
+  $my_fqdn = $facts['networking']['fqdn']
+  $dot_split = split($my_fqdn, '[.]')
+  $my_hostname = $dot_split[0]
+  $dash_split = split($my_hostname,'[-]')
+  $location = $dash_split[1]
+  $environment = $dash_split[2]
+
+  sunet::nftables::allow { 'allow-step-ca-acme':
+    from  => '89.45.237.248', # internal-sto3-test-ca-1.cdn.sunet.se
+    port  => 80,
+    proto => 'tcp',
+  }
+
+  # Get client cert for connecting to MQTT bus
+  exec { "certbot certonly -n --email patlu@sunet.se --no-eff-email --agree-tos --standalone -d ${my_fqdn} --server ${acme_url[$environment]} --http-01-address ${facts['networking']['ip']}":
+    creates => "/etc/letsencrypt/live/${my_fqdn}/fullchain.pem"
+  }
+
+  $sunet_cdnp_dir = '/var/lib/sunet-cdnp'
+  $sunet_cdnp_file = "sunet-cdnp_${sunet_cdnp_version}_linux_${facts[os][architecture]}.tar.gz"
+  $sunet_cdnp_url = "https://github.com/SUNET/sunet-cdnp/releases/download/v${sunet_cdnp_version}/${sunet_cdnp_file}"
+  # Create directory for managing CDP purger
+  file { $sunet_cdnp_dir:
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  exec { "curl -LO ${sunet_cdnp_url}":
+    creates => "${sunet_cdnp_dir}/${sunet_cdnp_file}",
+    cwd     => $sunet_cdnp_dir,
+    notify  => Exec['extract sunet-cdnp'],
+  }
+
+  exec { 'extract sunet-cdnp':
+    command     => "tar -xzf ${sunet_cdnp_file} sunet-cdnp",
+    cwd         => $sunet_cdnp_dir,
+    refreshonly => true,
+    notify      => Service['sunet-cdnp'],
+  }
+
+  file { "${sunet_cdnp_dir}/sunet-cdnp":
+    owner => 'root',
+    group => 'root',
+    mode  => '0755',
+  }
+
+  file { '/usr/local/bin/sunet-cdnp':
+    ensure => link,
+    target => "${sunet_cdnp_dir}/sunet-cdnp",
+  }
+
+  file { '/etc/systemd/system/sunet-cdnp.service':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/sunet-cdnp.service.erb'),
+    notify  => [Class['sunet::systemd_reload']],
+  }
+
+  service { 'sunet-cdnp':
+    ensure => 'running',
+    enable => true,
+  }
+
   if $cache_secrets {
     $customers.each |String $customer, Integer $customer_uid| {
       if $cache_secrets['customers'][$customer] {
-        file { "/opt/sunet-cdn/customers/$customer":
+        file { "/opt/sunet-cdn/customers/${customer}":
           ensure => directory,
           owner  => $customer_uid,
           group  => $customer_uid,
           mode   => '0750',
         }
 
-        file { "/opt/sunet-cdn/customers/$customer/shared":
+        file { "/opt/sunet-cdn/customers/${customer}/conf":
           ensure => directory,
           owner  => $customer_uid,
           group  => $customer_uid,
           mode   => '0750',
         }
 
-        file { "/opt/sunet-cdn/customers/$customer/cache":
+        file { "/opt/sunet-cdn/customers/${customer}/shared":
           ensure => directory,
           owner  => $customer_uid,
           group  => $customer_uid,
           mode   => '0750',
         }
 
-        file { "/opt/sunet-cdn/customers/$customer/conf/haproxy.cfg":
+        file { "/opt/sunet-cdn/customers/${customer}/cache":
+          ensure => directory,
+          owner  => $customer_uid,
+          group  => $customer_uid,
+          mode   => '0750',
+        }
+
+        file { "/opt/sunet-cdn/customers/${customer}/certs-private":
+          ensure => directory,
+          owner  => $customer_uid,
+          group  => $customer_uid,
+          mode   => '0750',
+        }
+
+        $combined_pem = "/opt/sunet-cdn/customers/${customer}/certs-private/combined.pem"
+
+        concat { $combined_pem:
+          ensure => present,
+          owner  => $customer_uid,
+          group  => $customer_uid,
+          mode   => '0640',
+        }
+
+        concat::fragment { "${customer}-fullchain-${cache_secrets['customers'][$customer]['host']}":
+          target => $combined_pem,
+          source => "/opt/certbot-sync/letsencrypt/live/${cache_secrets['customers'][$customer]['host']}/fullchain.pem",
+          order  => '01',
+        }
+
+        concat::fragment { "${customer}-privkey-${cache_secrets['customers'][$customer]['host']}":
+          target => $combined_pem,
+          source => "/opt/certbot-sync/letsencrypt/live/${cache_secrets['customers'][$customer]['host']}/privkey.pem",
+          order  => '02',
+        }
+
+        file { "/opt/sunet-cdn/customers/${customer}/conf/haproxy.cfg":
           ensure  => file,
           owner   => $customer_uid,
           group   => $customer_uid,
@@ -71,12 +271,20 @@ class cdn::cache(
           content => template('cdn/cache/haproxy.cfg.erb'),
         }
 
-        sunet::docker_compose { "sunet-cdn-cache-$customer":
+        file { "/opt/sunet-cdn/customers/${customer}/conf/varnish.vcl":
+          ensure  => file,
+          owner   => $customer_uid,
+          group   => $customer_uid,
+          mode    => '0440',
+          content => template('cdn/cache/varnish.vcl.erb'),
+        }
+
+        sunet::docker_compose { "sunet-cdn-cache-${customer}":
           content          => template('cdn/cache/docker-compose.yml.erb'),
-          service_name     => "cdn-cache-$customer",
-          compose_dir      => "/opt/sunet-cdn/compose/$customer",
+          service_name     => "cdn-cache-${customer}",
+          compose_dir      => "/opt/sunet-cdn/compose/${customer}",
           compose_filename => 'docker-compose.yml',
-          description      => "SUNET CDN CA $customer",
+          description      => "SUNET CDN CA ${customer}",
         }
       }
     }

global/overlay/etc/puppet/modules/cdn/manifests/db.pp

@@ -0,0 +1,76 @@
+# Configure a SUNET CDN DB server
+class cdn::db(
+  String $postgres_version = '17.0-bookworm',
+)
+{
+
+  $db_secrets = lookup({ 'name' => 'cdn::db-secrets', 'default_value' => undef })
+
+  if $db_secrets {
+    file { '/opt/sunet-cdn':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0755',
+    }
+
+    file { '/opt/sunet-cdn/compose':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0750',
+    }
+
+    file { '/opt/sunet-cdn/db':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0750',
+    }
+
+    # User/group 999 matches postgres user in container
+    file { '/opt/sunet-cdn/db/conf':
+      ensure => directory,
+      owner  => '999',
+      group  => '999',
+      mode   => '0750',
+    }
+
+    file { '/opt/sunet-cdn/db/docker-entrypoint-initdb.d':
+      ensure => directory,
+      owner  => '999',
+      group  => '999',
+      mode   => '0750',
+    }
+
+    file { '/opt/sunet-cdn/db/conf/init-cdn-db.conf':
+      ensure => file,
+      owner  => '999',
+      group  => '999',
+      mode   => '0640',
+      content => template('cdn/db/init-cdn-db.conf.erb'),
+    }
+
+    file { '/opt/sunet-cdn/db/docker-entrypoint-initdb.d/init-cdn-db.sh':
+      ensure => file,
+      owner  => '999',
+      group  => '999',
+      mode   => '0750',
+      content => file('cdn/db/init-cdn-db.sh'),
+    }
+
+    sunet::nftables::docker_expose { 'postgres-db' :
+      allow_clients => '127.0.0.1',
+      port          => 5432,
+      iif           => $facts['networking']['primary'],
+    }
+
+    sunet::docker_compose { 'sunet-cdn-db':
+      content          => template('cdn/db/docker-compose.yml.erb'),
+      service_name     => 'cdn-db',
+      compose_dir      => '/opt/sunet-cdn/compose',
+      compose_filename => 'docker-compose.yml',
+      description      => 'SUNET CDN DB',
+    }
+  }
+}

global/overlay/etc/puppet/modules/cdn/manifests/l4lb.pp

@@ -1,14 +1,120 @@
 # Configure the SUNET CDN l4lb service
 class cdn::l4lb(
+  Boolean $cilium_enabled = false,
   String $cilium_version = 'v1.16.1',
   String $cilium_devices = 'enp129s0f1np1',
 )
 {
-  sunet::docker_compose { 'sunet-cdn-l4lb':
-    content          => template('cdn/l4lb/docker-compose.yml.erb'),
-    service_name     => 'cdn-l4lb',
-    compose_dir      => '/opt/sunet-cdn/compose/l4lb',
-    compose_filename => 'docker-compose.yml',
-    description      => 'SUNET CDN l4lb',
+  if $cilium_enabled {
+    sunet::docker_compose { 'sunet-cdn-l4lb':
+      content          => template('cdn/l4lb/docker-compose.yml.erb'),
+      service_name     => 'cdn-l4lb',
+      compose_dir      => '/opt/sunet-cdn/compose/l4lb',
+      compose_filename => 'docker-compose.yml',
+      description      => 'SUNET CDN l4lb',
+    }
+  }
+
+  include sunet::systemd_reload
+
+  package {'conntrack': ensure => installed }
+
+  package {'bird2': ensure => installed }
+
+  file { '/opt/sunet-cdn':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0640',
+  }
+
+  $sysctl_file = '/etc/sysctl.d/99-cdn-l4lb.conf'
+    file { $sysctl_file:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('cdn/l4lb/sysctl.erb'),
+    }
+    # Load the sysctl file if it has changed
+    exec { "sysctl -p ${sysctl_file}":
+      subscribe   => File[$sysctl_file],
+      refreshonly => true,
+    }
+
+  file { '/opt/sunet-cdn/l4lb':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0640',
+  }
+
+  file { '/opt/sunet-cdn/l4lb/conf':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0640',
+  }
+
+  file { '/opt/sunet-cdn/l4lb/conf/netns-base.json':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/l4lb/netns-base.json.erb'),
+  }
+
+  file { '/opt/sunet-cdn/l4lb/conf/nft-l4lb.conf':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/l4lb/nft-l4lb.conf.erb'),
+  }
+
+  file { '/usr/local/bin/sunet-l4lb-namespace':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => file('cdn/l4lb/sunet-l4lb-namespace'),
+  }
+
+  file { '/etc/systemd/system/sunet-l4lb-namespace.service':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/l4lb/sunet-l4lb-namespace.service.erb'),
+  }
+
+  file { '/etc/systemd/system/bird.service.d':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  file { '/etc/systemd/system/bird.service.d/override.conf':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/l4lb/bird-override.conf.erb'),
+    notify  => [Class['sunet::systemd_reload']]
+  }
+
+  file { '/etc/bird/bird.conf':
+    ensure  => file,
+    owner   => 'bird',
+    group   => 'bird',
+    mode    => '0640',
+    content => template('cdn/l4lb/bird.conf.erb'),
+    notify  => Service['bird'],
+  }
+
+  service { 'bird':
+    ensure => 'running',
+    enable => true,
   }
 }

global/overlay/etc/puppet/modules/cdn/manifests/mqtt.pp

@@ -2,6 +2,7 @@
 class cdn::mqtt(
   String $dc = '',
   Array[String] $clients = [],
+  Array[String] $client_ips = [],
   Hash[String, Hash] $bridges = {},
   Hash[String, String] $acme_url = {
     test => 'https://internal-sto3-test-ca-1.cdn.sunet.se:9000/acme/acme/directory'
@@ -35,6 +36,14 @@ class cdn::mqtt(
     proto => 'tcp',
   }
 
+  $client_ips.each | String $client_ip | {
+    sunet::nftables::allow { "allow-acme-client-${client_ip}":
+      from  => $client_ip,
+      port  => 8883,
+      proto => 'tcp',
+    }
+  }
+
   # From https://wiki.sunet.se/display/sunetops/Platform+naming+standards
   $my_fqdn = $facts['networking']['fqdn']
   $dot_split = split($my_fqdn, '[.]')

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-dummy.netdev.erb

@@ -0,0 +1,3 @@
+[NetDev]
+Name=dummy0
+Kind=dummy

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-dummy.network.erb

@@ -0,0 +1,10 @@
+[Match]
+Name=dummy0
+
+[Network]
+<% @cache_secrets['customers'].each do |customer, customer_settings| -%>
+# <%= customer %>
+Address=<%= customer_settings['ip4'] %>/32
+Address=<%= customer_settings['ip6'] %>/128
+
+<% end -%>

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-ip6tunl.netdev.erb

@@ -0,0 +1,16 @@
+[NetDev]
+Name=ip6tnl0
+Kind=ip6tnl
+
+[Tunnel]
+Independent=true
+# Local and Remote should be 'any' or '::' but right now it does not work.
+# However, by using the name "ip6tnl0" that matches the name of the interface
+# automatically created when the kernel module is loaded it seems we can set
+# anything "non-zero" here and the config will be accepted but still leaving
+# "::" as both Local and Remote on the interface. This then results in the
+# wanted configuration for now. One bug cancelling out another bug?
+# https://github.com/systemd/systemd/issues/34930
+Local=::1
+Remote=::1
+Mode=ip6ip6

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-ip6tunl.network.erb

@@ -0,0 +1,2 @@
+[Match]
+Name=ip6tnl0

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-ipip.netdev.erb

@@ -0,0 +1,8 @@
+[NetDev]
+Name=tunl0
+Kind=ipip
+
+[Tunnel]
+Independent=true
+Local=0.0.0.0
+Remote=0.0.0.0

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-ipip.network.erb

@@ -0,0 +1,2 @@
+[Match]
+Name=tunl0

global/overlay/etc/puppet/modules/cdn/templates/cache/docker-compose.yml.erb

@@ -44,7 +44,7 @@ services:
     # Use the same custom user as is used for haproxy.
     user: <%= @customer_uid %>:<%= @customer_uid %>
     volumes:
-      - /opt/sunet-cdn/customers/<%= @customer %>/conf/default.vcl:/etc/varnish/default.vcl:ro
+      - /opt/sunet-cdn/customers/<%= @customer %>/conf/varnish.vcl:/etc/varnish/varnish.vcl:ro
       - /opt/sunet-cdn/customers/<%= @customer %>/shared:/shared
       - /opt/sunet-cdn/customers/<%= @customer %>/cache:/cache
     # From https://www.varnish-software.com/developers/tutorials/running-varnish-docker/:
@@ -65,7 +65,7 @@ services:
       "-a",
       "/shared/varnish.sock,PROXY,mode=600",
      "-f",
-     "/etc/varnish/default.vcl",
+     "/etc/varnish/varnish.vcl",
      "-p",
      "feature=+http2",
      "-s",

global/overlay/etc/puppet/modules/cdn/templates/cache/haproxy.cfg.erb

@@ -28,8 +28,10 @@ defaults
 	errorfile 504 /usr/local/etc/haproxy/errors/504.http
 
 frontend customer
-	bind <%= @customer_ip %>:80
-        bind <%= @customer_ip %>:443 ssl crt /certs-private/customer.pem
+	bind <%= @cache_secrets['customers'][@customer]['ip4'] %>:80
+	bind [<%= @cache_secrets['customers'][@customer]['ip6'] %>]:80
+	bind <%= @cache_secrets['customers'][@customer]['ip4'] %>:443 ssl crt /certs-private/combined.pem
+	bind [<%= @cache_secrets['customers'][@customer]['ip6'] %>]:443 ssl crt /certs-private/combined.pem
 	default_backend varnish
 
 backend varnish
@@ -43,8 +45,8 @@ listen haproxy_https
 	bind    /shared/haproxy_https accept-proxy mode 640
 	stick-table type ip size 100
 	stick   on dst
-<% @cache_secrets['customers'][@customer]['origins'].each do |index, origin_ip| -%>
-	server s<%= index %> <%= origin_ip %>:443 ssl ca-file /etc/ssl/certs/ca-certificates.crt alpn http/1.1 sni fc_pp_authority
+<% @cache_secrets['customers'][@customer]['origins'].each_with_index do |origin, index| -%>
+	server s<%= index %> <%= origin %>:443 ssl ca-file /etc/ssl/certs/ca-certificates.crt alpn http/1.1 sni fc_pp_authority
 <% end -%>
 
 # While varnish could recach out to HTTP backends itself, lets stick to one
@@ -56,6 +58,6 @@ listen haproxy_http
 	bind    /shared/haproxy_http accept-proxy mode 640
 	stick-table type ip size 100
 	stick   on dst
-<% @cache_secrets['customers'][@customer]['origins'].each do |index, origin_ip| -%>
-        server s<%= index %> <%= origin_ip %>:80
+<% @cache_secrets['customers'][@customer]['origins'].each_with_index do |origin, index| -%>
+        server s<%= index %> <%= origin %>:80
 <% end -%>

global/overlay/etc/puppet/modules/cdn/templates/cache/sunet-cdnp.service.erb

@@ -0,0 +1,16 @@
+# This service file is generated by Puppet. Do not edit.
+[Unit]
+Description=SUNET CDN Purger
+Wants=docker.service
+After=docker.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/sunet-cdnp \
+  -mqtt-ca-file /usr/local/share/ca-certificates/step_ca_root.crt \
+  -mqtt-client-key-file /etc/letsencrypt/live/<%= @networking['fqdn'] %>/privkey.pem \
+  -mqtt-client-cert-file /etc/letsencrypt/live/<%= @networking['fqdn'] %>/fullchain.pem \
+  -mqtt-server <%= @mqtt_url[@location][@environment] %>
+
+[Install]
+WantedBy=multi-user.target

global/overlay/etc/puppet/modules/cdn/templates/cache/sysctl.erb

@@ -0,0 +1,2 @@
+# Allow haproxy to bind to port 80 and 443 when running as unprivileged user
+net.ipv4.ip_unprivileged_port_start=80

global/overlay/etc/puppet/modules/cdn/templates/cache/varnish.vcl.erb

@@ -0,0 +1,92 @@
+# The builtin VCL is called when there is no explicit
+# return statement.
+#
+# See the VCL chapters in the Users Guide for a comprehensive documentation
+# at https://www.varnish-cache.org/docs/.
+
+# Marker to tell the VCL compiler that this VCL has been written with the
+# 4.0 or 4.1 syntax.
+vcl 4.1;
+
+import std;
+# https://www.varnish-software.com/developers/tutorials/avoid-http-to-https-redirect-loops-varnish/#create-cache-variations-based-on-the-x-forwarded-proto-header
+import proxy;
+
+# https://varnish-cache.org/docs/trunk/users-guide/vcl-backends.html#connecting-through-a-proxy
+backend haproxy_https {
+  .path = "/shared/haproxy_https";
+}
+
+backend haproxy_http {
+  .path = "/shared/haproxy_http";
+}
+
+backend destination_http {
+  .host = "<%= @cache_secrets['customers'][@customer]['host'] %>";
+  .port = "80";
+  .via = haproxy_http;
+}
+
+backend destination_https {
+  .host = "<%= @cache_secrets['customers'][@customer]['host'] %>";
+  .port = "443";
+  .via = haproxy_https;
+}
+
+sub vcl_recv {
+  # Happens before we check if we have this in cache already.
+  #
+  # Typically you clean up the request here, removing cookies you don't need,
+  # rewriting the request, etc.
+  #
+
+  # The usage of the proxy module is possible because haproxy is configured
+  # to set PROXY SSL headers for us.
+  if (proxy.is_ssl()) {
+    set req.http.X-Forwarded-Proto = "https";
+    std.syslog(180, "RECV: this is https");
+    if (req.http.host == "<%= @cache_secrets['customers'][@customer]['host'] %>") {
+      set req.backend_hint = destination_https;
+    }
+  } else {
+    set req.http.X-Forwarded-Proto = "http";
+    std.syslog(180, "RECV: this is http");
+    if (req.http.host == "<%= @cache_secrets['customers'][@customer]['host'] %>") {
+        set req.backend_hint = destination_http;
+    }
+  }
+  if (req.method == "PURGE") {
+    if (req.http.x-sunet-cdn-key == "<%= @cache_secrets['customers'][@customer]['key'] %>") {
+      return (purge);
+    }
+    return(synth(405,"Not allowed."));
+  }
+}
+
+sub vcl_backend_response {
+  # Happens after we have read the response headers from the backend.
+  #
+  # Here you clean the response headers, removing silly Set-Cookie headers
+  # and other mistakes your backend does.
+
+  # Use slash/fellow for storage
+  set beresp.storage = storage.fellow;
+
+  # Hold stale objects (where TTL has expired) for a longer time
+  set beresp.grace = 30m;
+
+  # https://www.varnish-software.com/developers/tutorials/avoid-http-to-https-redirect-loops-varnish/#create-cache-variations-based-on-the-x-forwarded-proto-header
+  if(beresp.http.Vary) {
+    set beresp.http.Vary = beresp.http.Vary + ", X-Forwarded-Proto";
+  } else {
+    set beresp.http.Vary = "X-Forwarded-Proto";
+  }
+}
+
+sub vcl_deliver {
+    # Happens when we have all the pieces we need, and are about to send the
+    # response to the client.
+    #
+    # You can do accounting or modifying the final object here.
+    unset resp.http.Vary;
+}

global/overlay/etc/puppet/modules/cdn/templates/db/docker-compose.yml.erb

@@ -0,0 +1,13 @@
+services:
+  db:
+    image: "postgres:<%= @postgres_version %>"
+    environment:
+      - POSTGRES_PASSWORD=<%= @db_secrets['postgres_password'] %>
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - /opt/sunet-cdn/db/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+      - /opt/sunet-cdn/db/conf:/conf
+volumes:
+  postgres_data:

global/overlay/etc/puppet/modules/cdn/templates/db/init-cdn-db.conf.erb

@@ -0,0 +1,2 @@
+# File sourced by init-cdn-db.sh
+cdn_password="<%= @db_secrets['cdn_password'] %>"

global/overlay/etc/puppet/modules/cdn/templates/l4lb/bird-override.conf.erb

@@ -0,0 +1,4 @@
+[Service]
+NetworkNamespacePath=/var/run/netns/l4lb
+Wants=sunet-l4lb-namespace.service
+After=sunet-l4lb-namespace.service

global/overlay/etc/puppet/modules/cdn/templates/l4lb/bird.conf.erb

@@ -0,0 +1,115 @@
+# Configure logging
+log syslog all;
+
+# Set router ID. It is a unique identification of your router, usually one of
+# IPv4 addresses of the router. It is recommended to configure it explicitly.
+router id <%= @facts['networking']['ip'] %>;
+
+# The Device protocol is not a real routing protocol. It does not generate any
+# routes and it only serves as a module for getting information about network
+# interfaces from the kernel. It is necessary in almost any configuration.
+protocol device {
+}
+
+# The direct protocol is not a real routing protocol. It automatically generates
+# direct routes to all network interfaces. Can exist in as many instances as you
+# wish if you want to populate multiple routing tables with direct routes.
+protocol direct {
+  ipv4;
+  ipv6;
+  interface "dummy0";
+}
+
+# The Kernel protocol is not a real routing protocol. Instead of communicating
+# with other routers in the network, it performs synchronization of BIRD
+# routing tables with the OS kernel. One instance per table.
+protocol kernel {
+  ipv4 {
+    import none;
+    export filter { if source = RTS_DEVICE then { reject; } accept; };
+  };
+# Create ECMP routes in kernel table from multiple paths
+  merge paths;
+}
+
+protocol kernel {
+  ipv6 {
+    import none;
+    export filter { if source = RTS_DEVICE then { reject; } accept; };
+  };
+  # Create ECMP routes in kernel table from multiple paths
+  merge paths;
+}
+
+protocol bgp tug_r11_v4 {
+  description "tug-r11-v4";
+  local 130.242.64.233 as 65443;
+  neighbor 130.242.64.232 as 1653;
+  hold time 90;
+
+  ipv4 {
+    import filter {
+      if net ~ [ 0.0.0.0/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 188.240.152.0/24{32,32} ] then
+        accept; else reject;
+      };
+  };
+}
+
+protocol bgp tug_r11_v6 {
+  description "tug-r12-v6";
+  local 2001:6b0:2006:74::1 as 65443;
+  neighbor 2001:6b0:2006:74:: as 1653;
+  hold time 90;
+
+  ipv6 {
+    import filter {
+      if net ~ [ ::/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 2001:6b0:2100::/48{128,128} ] then
+        accept; else reject;
+    };
+  };
+}
+
+
+protocol bgp tug_r12_v4 {
+  description "tug-r12-v4";
+  local 130.242.64.235 as 65443;
+  neighbor 130.242.64.234 as 1653;
+  hold time 90;
+
+  ipv4 {
+    import filter {
+      if net ~ [ 0.0.0.0/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 188.240.152.0/24{32,32} ] then
+        accept; else reject;
+      };
+  };
+}
+
+protocol bgp tug_r12_v6 {
+  description "tug-r12-v6";
+  local 2001:6b0:2006:75::1 as 65443;
+  neighbor 2001:6b0:2006:75:: as 1653;
+  hold time 90;
+
+  ipv6 {
+    import filter {
+      if net ~ [ ::/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 2001:6b0:2100::/48{128,128} ] then
+        accept; else reject;
+    };
+  };
+}

global/overlay/etc/puppet/modules/cdn/templates/l4lb/netns-base.json.erb

@@ -0,0 +1,20 @@
+{
+  "l4lb": {
+    "enp129s0f1np1": {
+      "ipv4": [
+        "130.242.64.233/31"
+      ],
+      "ipv6": [
+        "2001:6b0:2006:74::1/127"
+      ]
+    },
+    "enp129s0f0np0": {
+      "ipv4": [
+        "130.242.64.235/31"
+      ],
+      "ipv6": [
+        "2001:6b0:2006:75::1/127"
+      ]
+    }
+  }
+}

global/overlay/etc/puppet/modules/cdn/templates/l4lb/nft-l4lb.conf.erb

@@ -0,0 +1,40 @@
+#!/usr/sbin/nft -f
+
+flush ruleset
+
+table inet filter {
+        chain input {
+                type filter hook input priority 0; policy drop;
+
+                # accept any localhost traffic
+                iif lo counter accept
+
+                # accept icmp
+                ip protocol icmp counter accept
+                ip6 nexthdr icmpv6 icmpv6 type { destination-unreachable, packet-too-big, time-exceeded,
+                                                 parameter-problem, echo-request, mld-listener-query,
+                                                 nd-router-solicit, nd-router-advert, nd-neighbor-solicit,
+                                                 nd-neighbor-advert } counter accept
+
+                # accept traffic originated from us
+                ct state established counter accept
+                # silently drop invalid packets
+                ct state invalid counter drop
+        }
+        chain forward {
+                type filter hook forward priority 0; policy drop;
+        }
+        chain output {
+                type filter hook output priority 0;
+        }
+}
+
+# HTTP and HTTPS
+add rule inet filter input tcp dport 80 counter accept comment "l4lb HTTP"
+add rule inet filter input tcp dport 443 counter accept comment "l4lb HTTPS"
+
+# BGP
+add rule inet filter input ip saddr { 130.242.64.232 } tcp dport 179 counter accept comment "tug-r11-v4"
+add rule inet filter input ip saddr { 130.242.64.234 } tcp dport 179 counter accept comment "tug-r12-v4"
+add rule inet filter input ip6 saddr { 2001:6b0:2006:74:: } tcp dport 179 counter accept comment "tug-r11-v6"
+add rule inet filter input ip6 saddr { 2001:6b0:2006:75:: } tcp dport 179 counter accept comment "tug-r12-v6"

global/overlay/etc/puppet/modules/cdn/templates/l4lb/sunet-l4lb-namespace.service.erb

@@ -0,0 +1,11 @@
+[Unit]
+Description=Setup l4lb namespace used for all service traffic
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/sunet-l4lb-namespace
+
+[Install]
+WantedBy=multi-user.target

global/overlay/etc/puppet/modules/cdn/templates/l4lb/sysctl.erb

@@ -0,0 +1,5 @@
+# Allow non-SYN packets to be accepted for a IPVS service.
+# Needed if another l4lb has been taken down and routers are now sending us
+# packets for existing connections that was previously handled by the other
+# node.
+net.ipv4.vs.sloppy_tcp=1

global/overlay/etc/puppet/modules/cdn/templates/mqtt/cdn.conf.erb

@@ -1,6 +1,6 @@
 listener 8883
-cafile /etc/mosquitto/ca_certificates/chain.pem
-certfile /etc/mosquitto/certs/cert.pem
+cafile /usr/local/share/ca-certificates/step_ca_root.crt
+certfile /etc/mosquitto/certs/fullchain.pem
 keyfile /etc/mosquitto/certs/privkey.pem
 require_certificate true
 use_identity_as_username true

global/overlay/etc/puppet/setup_cosmos_modules

@@ -83,7 +83,7 @@ def main():
         "sunet": {
             "repo": "https://github.com/SUNET/puppet-sunet.git",
             "upgrade": "yes",
-            "tag": "stable-2023v1-2*",
+            "tag": "patlu-dockerhost2-ipv6-nat-2*",
         },
         "augeas": {
             "repo": "https://github.com/SUNET/puppet-augeas.git",

global/pre-tasks.d/020common-tools

@@ -8,7 +8,7 @@ set -e
 stamp="$COSMOS_BASE/stamps/common-tools-v01.stamp"
 
 if ! test -f $stamp; then
-    apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades ntp
+    apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades
     update-alternatives --set editor /usr/bin/vim.basic
 
     mkdir -p `dirname $stamp`

internal-sto3-test-cache-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml

@@ -2,5 +2,38 @@
 cdn::cache-secrets:
   customers:
     customer1:
+      key: ENC[PKCS7,MIIDCwYJKoZIhvcNAQcDoIIC/DCCAvgCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0xLmNkbi5zdW5ldC5zZQIUJEYAJSDlgRqHR7z1XpIgwd3vL7gwDQYJKoZIhvcNAQEBBQAEggIAhHA91YylFORnQ6q/5m10Vokw5NY+dWyF8P43p0Qr34Lr9eOqw5lMZDnZxbgZDQr7fJPsNFxQPF2pVIa1cvHiPW0an7k56VJnHoseh7TlNNa+wx10alSuiuVFEPUoNo4uSD3hjcjWUzQygckgtqqk9PBG9CnWVevLG1cpFeUCCNa+HD/cf318uajyItiJn4EzB5Lyd7b7AsqKnEdkomPnydwMXGObeV09WY2KJ4wzIXKtAUvzLj5iQ3j9sib/M7tx18JZvilKW0AOk+diQ06Lb09iIRTxBDMrPiEARzHyCsp0kGxEAoAZQllrnA/oU6z8S/FcWxo9j+loY2MLBQ7xlN/nL2sMWTjsV5XWSl8FeoKrAAcQ+1U8hDctNfvkMSgIjilah3789qY2PU9HgT1hY0XtVLnS6n9xH6ZeV82X36Je6YTFcKQSITnJvb4AH+8uowvRgYb7FyFiCRgGlGMxByEhfsr/XOjTzTgHZx+cSoJO/5P++7RGz6mU0pBP3J6/QVvJl/7DGyoetaSVkOUwOdDpeWs5N9QTGxQSVWQM9ermZCAjXJsQCG1hGg8jbr9J9pFFKbuzKNJHkpxt0euIuGs102SUW3+INLaVoiOzsJHj1nKsQWL0EE67SPt14f9RZ6vWLiom6lKnf39EIOgeFMCdEW7/MEfsNpiySyIwI5QwXAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQQlYRoAB7KwhhbDxUmCIza4AwB9NYi+5Zz8c3OM/7GPp8tXtGcfmKjoDb5z3Yd84u4E1qjm82zHLwxMTavowSRiUk]
+      host: ENC[PKCS7,MIIC+wYJKoZIhvcNAQcDoIIC7DCCAugCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0xLmNkbi5zdW5ldC5zZQIUJEYAJSDlgRqHR7z1XpIgwd3vL7gwDQYJKoZIhvcNAQEBBQAEggIAifVPgYnF+hjviovROhUs9DlGZM53Uf3FjcIgTLRJlQ+pVIq01OuZZfGW1q2heLtV/wHb7al1Xfbq6FH5RJFojqDqjCvn485GfTanx3tk+OMytcS1ACjTX4XOTGLtBlgKZXv888NHp0H9xUwYiyhZQ9l26EXZhlUW6OQ7O0OSM7cwBUY0B/bW1RBjq4bXqQ6cm7CXfkRhuD9V7vYVTmW4hpl09P0fbNJIAzCO6LZpI1x4n4hvLAyDvTphFtVt/wCBPORq/a8Lr7tuD3H0wTQCwHWN1G9q1qSl9PXjpknt77tA+ykHltRh71tKbl3izLT6ytYhWrlPbGjCuytnSV63+UXYKj5UjqOYBq/UR6aqTwY4DsrhRQMmMMUxlF2ibYQMd0l9ZQCliXMOT1OKzRVCigpTJQl4fnxV62+RSgJPBAboiR9uGwk9q6Z1eOroWVVPV0Qq68Tn1fvzMnU/HQ+425Mg7cH35NWsD+EIRhpi2/8F5ONd0P74DsDtTuxi7q7mJ4ZP7+n6IYUdoGRHSAnJSOaZqjyJWmlQk6ZntJTDYyC7bKamCML2Tz5D7q2pm5HjQeqcRthgX/PaIIwEi9cTuPyyD3funJsvJtoNH7BspI0ncAP4CZORHwNPBSFsVmCFw2Rv0KojysgOHWr3Uymjti4Lq6SGfEAxxMtBRFUFCaUwTAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQL2hoIWvXnGnfl9FV/rc/g4Ag5jueWu8mguigFjLXPB1SDcYv04NKYGW8addXEwEXoqA=]
+      ip4: ENC[PKCS7,MIIC6wYJKoZIhvcNAQcDoIIC3DCCAtgCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0xLmNkbi5zdW5ldC5zZQIUJEYAJSDlgRqHR7z1XpIgwd3vL7gwDQYJKoZIhvcNAQEBBQAEggIAuQFDR7/DHr0+W+iFLFaPHrsb/yxnl5ZjGXL+eNT8I4/JiWEw7ImL6kgNXOz/HXPt9GU9KxNc9K5wPprZ2C+0W1EMNue4nFFgoVRmZ2ZQERCkAxCbnIMD9StkTEWXaey0IUIyFRv1Z39nezzeP27Tf2opUsu44uEFhjLR+bGsHMvoAHU7ojPCfYjm8OCWwLoPab2ZoBhR9ltOp/EvEfJHEU5FF+zcRESLxm2BKG6X86LTM0KAZ7qSesmDTsxEWp+oIh0lxdexd80j2886QIpN/U6fQFLnfTZ/Z1L/6m4lMze9J/mvuv42wZdZI2IDAy0lA+ZBJgdYRyowzn0BkAe3MwWhnMrYb4G6O3eCSyQ0vz2yrXTHtRDPAj/e2YtluanfI85Y0l3rlLzMnc6FGzHxqcecj8Fv7dYuOZ7ieWynlJe0D2MBHQIQxyEPpihzp0vY9prqvsooxv2NymdQ9piqCH2QyPDdXLYPgbY8dY3HOjJaPVo+PoGXXcB64ALEFbko+OFutZgOtrfLNMYXZcLXD4/w3kC8ZdveWDscoNbrJzLbO9OoMzsh6Rc3Og7yzOYVRzoE+FfOGr9XVmmILxOSAW7v2/j1eOyJxiVNXtZeOT1EE/UpgvPJcEfx/REoRn1r3aThEr4Yrg0AHPpvyfm5hf/kQcbjVtOBxoW0ul+KwcowPAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQpnJ46bsEbyjtb8RtPt/LXYAQHd5jWNsEaGJLpqxGKEbA6w==]
+      ip6: ENC[PKCS7,MIIC+wYJKoZIhvcNAQcDoIIC7DCCAugCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0xLmNkbi5zdW5ldC5zZQIUJEYAJSDlgRqHR7z1XpIgwd3vL7gwDQYJKoZIhvcNAQEBBQAEggIAljd6fcqDuioxqTrln7Dq0JTkaQuGwX5RnMWjEkWebbRdI2lQy9eb/nkfMgMFGb8WXpdJ4hBXKHIscAGRcK7NDUWOgxuhkMmT1wo1C9pJjswaJscHJqdZcr2xRTVK5xB/qfI/h9daTwqFu/m9ZGDARSAWguizI3OfGTVFYpTPI29zuIElIp+wyraTLVnqzB2tM0KNwibVSyl/BBedHkjpYeRHO6ykIzJZxZK06lgGykF0Udz2v3QSZJ4vyHWCXuBdWJsN4chDQkPvl84egjzd73OL0WfQoE4LrfmcmDaZu0top4ZwkreaLZA9z3TenFd7IUlIB7xyGrSDoItwS3LZJ6vTKrQeP0DTq9y+nqcNlFeqNgs5VCGF4p6DX8zwWStFohYXSp6kgxYe5V7KUpHeVUQYZNervoajb7bySYilP6ykgK8QT3Q70dsUQBaWdsCWW9p6VdasjI027vYiPCXD/7bliKLGjDdeBGR4S/UgFNsTzGnfV/PWn7SHN4GIbRwxXgLl2ZJMeDX0TJ9YcIn4Vs7HcB4LUuGobvMcJFjgZk6nj9Mc1pdQfCMl/P7r2ZA5Qt0bqlc/jeYRVRGRXgI49xuHSvZTuFnSvwJndVqQfMFgBgcTwFKnIyX2diKESGc9z/9A3LmSNf/2duG7xH3uX/wDqx88gnmcWOo0ktnLNQswTAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQRifWemfoVp0jTnrORmuzjIAgvAw0cfOOB3AfOZMrQdJaVHaDt3yGxC/XcVk5x4L8ri4=]
       origins:
         - ENC[PKCS7,MIIC6wYJKoZIhvcNAQcDoIIC3DCCAtgCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0xLmNkbi5zdW5ldC5zZQIUJEYAJSDlgRqHR7z1XpIgwd3vL7gwDQYJKoZIhvcNAQEBBQAEggIAlcofAryMHgzdzYKeslCB7ekukxm30xOyYVHusJ4mwylU82xZjtXpcneoPJKydc2bSSP6LkuGGSaiziY1XdU/TLid9fhfBtSFM1S8Cc8IoAgZd25wXo0eaf2f5AfDcSJkywW+kfySEmXCGPGaCeo25k8zZDhWvI4ZrNBDgCTksuL99HWjsT6T6orC9qu2GEBWSpXcKbKu/Yv/1XbNvsJpW1XgwkGJz2EXjJSLKljhBG4P7i8tHIGHbJn/Xo6IWgsBqi8elc5sb/gK4LE5k+JSv78M7ZRs7aRaBm25LZnvNKBPCpyy9sgDEbJzGsHOTtfuyPpgra34wVy/Vt2XFOFserbg/4ZBjkCEN1pNSjb4jDh1tedlvjV1Ovf4W5EQcE1eu2UWvyfLLjjPHz0nW2OQbJB1zO6XOEdP2ot4+WnSgsZvCv9kGJPRhbbrdLVwW/5eSttbaWPo0viNavvTSgjYBiJ38LhPmpulHzhsCSFZisjq/ue6oFhoUwHb/pMklGQRJ2i2AqzPECstCTuKpSMTMrlnAgFS7u3DDXjmeYlV0UlF1SlEpGYqkGQspfWI+7DAnDomVDGAL8M6/+xpaudTNbcTvbrbPh0kwUjKR4+cP5mUAbNEho5UxI5tGnIRWRRd560+FKrAKPY/0V/Dx/tVpu7FgrgOEScdExMzAfDrTSwwPAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQz45S0wVsGWF9YXUE7fQwd4AQPn+7I2vxK/66O56enFIfQQ==]
+certbot_sync_client_ssh_key: >
+  ENC[PKCS7,MIIEnwYJKoZIhvcNAQcDoIIEkDCCBIwCAQAxggKTMIICjwIBAD
+  B3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRV
+  lBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0xLmNkbi
+  5zdW5ldC5zZQIUJEYAJSDlgRqHR7z1XpIgwd3vL7gwDQYJKoZIhvcNAQEBBQ
+  AEggIAWXfHKGu8/x9Hgb2AG+5+waywkR6CBxlLMEBo2NVdqhjRFCE3vPeTeJ
+  j4Th81KR0RSI42U9/wCZMF8Dq/SSybfTf4CiEOo3xziXdH46qNqdGjouvLx7
+  fA2l8YnMQo+Wwv8NNd3N7u+p4yJ6/3Dxcw3WzkcU7Igsk2xC2AZBUufu78+a
+  dAaMZwWtOcszOUHrSzen40BiBWBw36wk3aj+Yu8JD3X3I/wZm9SuiVp7M9ZZ
+  YDyX7VxhizdbrC0+/KOexla7hnLtzPfEHzrO1rOf3Yqz71LcWFjbF2H3HCOj
+  3cStdEJ5OeN6U4DYLXy45yxFTHSvph+X22uKtgib9aUGANb7OK21uI6WjJ6X
+  MN7Lg3kQNBmmNTGELbMIQ3aFKNiOmIqKPaNxAPxUwCCVA42bLJ3aNEH3AAMM
+  FId8+uoreTBZWxp/zw/rZe+rKgruBxeyLNKECcP+EPmMnD/CutbD6gLqDTzV
+  cSZoBscqWqmjZoRdXSR1w+tAvSChTDRCagsWD6Cwj3Bqs6Dbf/aFC7VON6lO
+  nS/yA1hiAFC6p99MtZMM/UYYgr8XGll53X2g3d/fq4kiN0RCf2DSR8xRVyNp
+  HVq5M8ZZUvHlB67nbYxPHPOyOxPpjeBOwT/9OpVi3frr2AaUZNZTzAM2dHKh
+  p0YL9q+BZptn0WKAzjdVTkbMUMhvwwggHuBgkqhkiG9w0BBwEwHQYJYIZIAW
+  UDBAEqBBBAz7LpDqW08uEZwSrdu0s7gIIBwKhSSSqj3Zg5OMbHp6cus2D/Ca
+  Y/hp+v3EVddS0CYYNR4L4I4cqk3MT1EeCpUyyUN8mUZD5+jG+BrHuJfWOZPU
+  SAfp6VdWES/UYRZGNi2iSbpJzuQaoENX5kqE1W0fF+V7I7yfyFUxkZRGtpE7
+  4TibdvUR5VycZ0O31iOUDwkXJaIGD8LYMUhc94oG2zEb3hNyAfhEleRKSHbY
+  NZXv/Ghc9EfbX9grnlGeudyved15rCv60aMU+NsWnVsrlX2s0W7ewBkVJDCd
+  8MYokAYlOFaGzn0nBazb7yYXL5aJuy+V9d3tqjmOtN1ic4KqXZaOQSa6B4xx
+  pWzeu/TFgGj7/fz7NfPeufSwU1eoObi2iTZ39jKKcu4jFerFLy6/0J71kyZd
+  89YQtbAjGOu1/PfV3R23SLV95CP2WrYIvG9Cxu+lYD/WSM23sD5BHCvAdnWH
+  DFwrjPGkZb75YOy3VJwz8kAbzkqqQ52V9CngFACuPMNOfHkUC/eRRFdeYHyp
+  JQ6GEnjPOStWZ3pAk/XqCOngh4wQEsU22goya+J0LpeBpMi+mcXsRC8h519l
+  Ulqr+/zsn8tIra3YpLSjAa6RqTpJ+R9Q8=]
+certbot_sync_server: internal-sto3-test-cs-1.cdn.sunet.se

internal-sto3-test-cache-1.cdn.sunet.se/overlay/root/.ssh/id_certbot_sync_client.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICwQG6Z0+xcV2HribPifp0MXXY9GLdISS9dHF1+qUdlw root@internal-sto3-test-cache-1.cdn.sunet.se

internal-sto3-test-cache-2.cdn.sunet.se/README

@@ -0,0 +1,3 @@
+
+The system documentation is in the docs directory of the multiverse repository.
+

internal-sto3-test-cache-2.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml

@@ -0,0 +1,39 @@
+---
+cdn::cache-secrets:
+  customers:
+    customer1:
+      key: ENC[PKCS7,MIIDCwYJKoZIhvcNAQcDoIIC/DCCAvgCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0yLmNkbi5zdW5ldC5zZQIUdz1N+zcs0PBGXqzkGwE/CGSTG5MwDQYJKoZIhvcNAQEBBQAEggIAI1BuG1SsQnLmrCgY+Sy2Vl0DslfUWntWhgmy+VNiN+DsGqJM1u46LY7KHCUstssofwafq8OFaSFRc22L63VQaHgY/LYMV/6OwvNWsNb7H4h6giXKTYbqVDwU6+sSvbGA34Yk/YdodPTHmMu+fJVXNAiVIcmUJ2KvPBboiD89lLwwyREP98hFF5tjJm91EuuS4q8msatrw7hB2R/Us1SWp+Jb4Yqh2h4hvkXtVMjrzbjDzZYSdgBm/3uJAvFaEkI5f43DLcv5I9ZZS+eX/Yame7YMheR8Kcc67RfTqiAmqFCNltLQ1yMTMDHICuCay+chuah/SYVrXilbalBxqtiTtUBkJ8LW2DwvHfQXvhtcpkQ/4wMXsSIHO4qLvUbIgze/T+zreUSgJYcqOSymYJPGM66G7mfsva5FDlEfIz15Cia9+uF7LQcw6e1vJitSy7bbL+I8HHoX+KhlfGC7I21+OBTF8j7Ju+Qj11zUa4WmZYRQxdirhJCKaD1djA+oYOffqMZ3zFD8CQidJ+uxGLTCrCR5A2zs4Z4gasWvETmrRwg6Gv4wBcx3908euruHJ7scYk4pTJWCySa5INsD6uNLWBPcspfXd/PEmFgp8SjR8lCkmoGcDgzx6AkuT/wf0w9dUmdmbG6p+pBfXWR2ROhWHOHvj/C8vHZJb8XivMuHyTowXAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQN81EZkXnsIsb1Lrj4J8Hd4AwdJUyG/D8DCU8fMySPfIHTHk6f5EFMDNPYhyO71d0zeDusHXuNWF9dxBjtGhezG4k]
+      host: ENC[PKCS7,MIIC+wYJKoZIhvcNAQcDoIIC7DCCAugCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0yLmNkbi5zdW5ldC5zZQIUdz1N+zcs0PBGXqzkGwE/CGSTG5MwDQYJKoZIhvcNAQEBBQAEggIAn6fTKq6YOMlpahab4milaa/CYsIv9sWj2hTLeVtfRo0VaFTI2BVIu/2w/1Rk2JAYSPo3M97ToNOA09ZdmsgFnLwBXt54QeSrYQ1qzN8XLvcZrp+1FHvrmjSlB4VnKCq/Kp0Z0XzNVJ5pDLniJyXSV2efhzKqifWP9feMFvsCXgdpbJXc0+arFHBRN8CcBJvlJde6Nq5LzTWHN2NXrIOr3m8vLcPnsQEX9SFKy7tS9l6YXnxBmLmkvHciJHgFNzuIu2gfpuDfRpNjo5UGTeEhv8tMqyhYqPBNE/OtmycacGS4fd3juab4XMfT+HM7WV84khQyejJCDdhxvIIT8XM5mDYVS2z+qhnkqRugJEqo149jAtnXYQRSQgS4Cebj7oYaYFjhpxdALUrK4sPxxFqPewmpIJdtVUpmU8UjiwdZA3yQnnKGQby2hgOZN2mekbqWvzaNdxRwCakqeTESHJ/J0aedUz1fPNzaydbIJWK68oxmzHU2XQgQE1D3B1NJWldKvt4FOkzQYyouBltoyzcpRAScQUF5ty+mvfI14+omKDiecWwJMdzColjKML9X76aPdsSgbq5jw8tjjI4D+VXSGLU6xP1DCPZjLlj0TP1yhVxxOPkt+VtxO40jzM/HJ/yHDuS/BWP4z6KxOQRJnpGUFg0AFQk0tinQ1IUmG8EuDxIwTAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQ3ky0SRUp0ChVrTTCkWAKKoAgkFfDRY4ia37Jc55b18VfTvayazimefkS6sq7Pp7TlHg=]
+      ip4: ENC[PKCS7,MIIC6wYJKoZIhvcNAQcDoIIC3DCCAtgCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0yLmNkbi5zdW5ldC5zZQIUdz1N+zcs0PBGXqzkGwE/CGSTG5MwDQYJKoZIhvcNAQEBBQAEggIA4DpAdWaE1EvqEpHG03q+F4lc69CY5T93xZ/DGzr82aMbU2DyP0UJ9eKxkdn6Qak0wHFfIl2C1qy/RXw9T8/6YABGop4VL2Jlu74Qz+Jv+9hbiBIfLvmLYfY+s5MhkcLGtKmcdTwVKHGuki7NWPHp3iX9LFggJhWlQSdEuoJ/OFwzEiK3H5DxxitvBURPoicomgWKbRM0iAqV1iM3nNFUB/6cbTAb/WHwuA+K/AWBhTIhuB4M5lTfBqPoGBUDkHepZQJ7dXdax90y9Avgrx5rxnLtzhXcdsuJ+lPLbHYrEfOvImncf4l/Px51pT3FxkwZJpF0qgUqHgnioGb2KXpL1pIDL3ePtLTsFcM8gMSbG/d04BZ2HGHzK09e95fgF9HgTN6GZo4PHP+eWpdyR3UGNylWH6PVMuzyEA+UgrxcGMmqXNsMWgT8hjVDj5D5GUxaGmS4DnbgoWY7QKSHsKQCP2EN+h2nUDWai8AJ7jyjRs2eJ0yoKtobmgcMkrfh4osnaQpjNiBxfE/D8l/VSWCvKmv8/ETG5c7nRSztaUAtvq8ysEBNaAyPuB8g8Nu7z3s2Hr/p7hfMzM0VNZY4b7DlvGmPqs5Av0lZV+IWK/t69DMyjlWplKqNoMZ87f6t7WxtGjvbUXhHYUwGQZMXWuaLiAtmaMYxYwhqQtbLykbPIRYwPAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQ1TMTF9KdFFxqTTCw01QcuYAQz26QlHGb4gPFOaByg8YpyQ==]
+      ip6: ENC[PKCS7,MIIC+wYJKoZIhvcNAQcDoIIC7DCCAugCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0yLmNkbi5zdW5ldC5zZQIUdz1N+zcs0PBGXqzkGwE/CGSTG5MwDQYJKoZIhvcNAQEBBQAEggIAP2FzvJqxJUktd4ydtawRbHZmbgQ0Db9k6E09c3YC9VgSBfBd5IhUjsf7smoPRGoHywoqW8MthPZxH/QQ7iwTAZuDCLRZ6SOlyrF6lfuH4QSUltrERjn2xntdyCiZTTL8p18mdwVK4cwyYv+aG7ztpC4aP+SahJ56pCwWSl9dA/695r9MI6a2Objf6E+biu92s5l5i8fPMleZU3en39lJ8XfiqG/z1u/Vf0f3HoGrsVeOt9rRuN6jFHRPomiFnz89kx2/UypcNU1P5MoC/q0t4dO2BcoTjcmDTwHuPzNWc8ukmW1JNNgZX2AAmjuOR2XHHOU2FijPiIni5opISFPi8lc1QNu1ptPn1OqruR5deSMJiDF/jRNZSk3RSBhoG4SXpxuStW/jmmMWdUILSn1GnYIqA0sPJvXqpeGDRRsqft+crRFnh13z8Uw/QDmQY+voRVRjRrGBD2uGnyX7QE+Ju+FP20fwdhLGdCOoi0Ea9RCJxFFWfLtZVAkmkUDbUI4xUww+Mkv8ITvbr73agBEPaoCdBc5hjzmaj+e/ZatH51p+WtY6rG+IWG441kpW4B1E5et2MQWqxCSEY82IrnliQ1LBOut9jjFLfe/UuoiddmOeBmYtsiCR7vxURDn8kG+wHvmfnAr2n+EDkf+2ojWkP9N07R8pSM+yBKCB9tTrZ38wTAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQj7eyNfGf3w8lSc1fh6PG0YAgWEtO8a6fOZPuxD6o/C5XJDmQHewD7NykrK1qiF5Tjdo=]
+      origins:
+        - ENC[PKCS7,MIIC6wYJKoZIhvcNAQcDoIIC3DCCAtgCAQAxggKTMIICjwIBADB3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0yLmNkbi5zdW5ldC5zZQIUdz1N+zcs0PBGXqzkGwE/CGSTG5MwDQYJKoZIhvcNAQEBBQAEggIAwrENSsgv+k4c96RjXy2Ps5p+l0qgiCouAgVMNQg3Tusrr1kbjnhsb0NpKE46Yiulm1Zw7aBpeXQzR2BVIgGo9uWu/7TF2P7CbMzZAKW5YEu53ccAwN/BRcunNlJTptxwaSRsHcv8nrO3Jo7vcfxN40pZDpWSlFcSu5xHHPwWHyKrERvjwl2wDjbBvRtDIfv8OJAx68J2kMrC1PcZ3WBK/tgO9IMJIFp+lKq+qwBx5Sbtckuyjz1L95iynIjQfetWqrF8SB0Svv80BRbdx4ONSrpgHH1hwplmWI+HlIYtMOkOZisQYdzWu39XloQHYq42fn6nt22lTx7ZTAR913ZFgH7KvUY9VRmq5zwYD10veY2QqelOhE6OE4BBmF0nRDzLRY+QZl8ancCkWNkbKlThNqtsoNzm9A6pKXlsJw/MdwDPxm0WE/m75BjXzObovisWp7yY9zLqJv2CdGtWuaXDQIXLPaCntXX8ZaEidSpcgnjXF9sfnknzvqvPibws3TQcyzTaemPYFScGX1Dp8wzQB72TJhusNUB1NyzRHApPlXB8PKLBlwR+hxogIaoq5HSmLmtkyi9wT2Gs8aC3dbvZA+OBgEdEi5Z2WcurUI4dFcyiKiSUDdRCjHWwiOii6BYvxZ7FCwifqeyRap0MvOstKGE47TsUSxwVS4zB/l3hZTswPAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQd3iYJWuQ0pbXRzv0mca4+4AQ8aZp4Ri24FUXRut6lhxkNw==]
+certbot_sync_client_ssh_key: >
+  ENC[PKCS7,MIIEnwYJKoZIhvcNAQcDoIIEkDCCBIwCAQAxggKTMIICjwIBAD
+  B3MF8xCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRV
+  lBTUwxMDAuBgNVBAMMJ2ludGVybmFsLXN0bzMtdGVzdC1jYWNoZS0yLmNkbi
+  5zdW5ldC5zZQIUdz1N+zcs0PBGXqzkGwE/CGSTG5MwDQYJKoZIhvcNAQEBBQ
+  AEggIAINGGnOJTuI8d+uB+J/3cTogIZ8ZGmynF25X0wMHgJcxyic7ONyfHuH
+  c8waNdHN4Lm5uTboFbSbcc69/41EqHNTrfQfZEhEn6hLHXbPXOLtQwAhWJeZ
+  CLH1p6UNFA7ZBngsgwENEmOqr0ZrX3fiv/yLKQwPbmQ8QdoD+SNIOVjy65Mm
+  0AgXDEL1K0XRtqsZnS9Vsvr1DJc105rT4JWCGbJGEJXvVSWdk1uKfux3SdUl
+  XsOCwELdChPANurjSoYkkMBKcPJ8B4ucvheXTB0fqA4euxy8JVkIjmadi9KS
+  Ccua1NPifBePhRP57N0PCSdR7HIhHZ/vLXps8FN3NYM6mH9WNnX5+7ZzL5ad
+  egNjdgIRIvo3Nhzx2p7Nq3oAnF+Tqsx50/dg7o8xhVojos+V2Mp8t/f6xnmQ
+  xK3/WCgygdO0+5Nu6sMQX3PhWUAt2n7ZSHpzKCHhxPcBG7XnBwjbHgCvcb+4
+  BTXHgAzSgmWfiTpcSudh/M6ZdSyndLL/EZYGUkWUu5A8W0quBJmUOa0H47Nn
+  JdAbZSKGkBfzJnBwQ9Tk4D6+UteI7UXpzHsGokzkNy+XDwELIKwbAy3ko2rQ
+  k3B3LryMtvFNL/BUvT+TCcFHOFMa/dciPPDDtTHFxZrdJrxeNOe4ozeQiQe6
+  wYuFY4jrRy8q0V2V4UdumtHKz9Wr8wggHuBgkqhkiG9w0BBwEwHQYJYIZIAW
+  UDBAEqBBAQRjMsWP7S0C6vH6483EKUgIIBwLHsyfda1R2PTM0lrpDllyrZfH
+  egF6lLdS9pwUHle0yukAAB6x+cn21/WDraZVIIKXY/rKqApk7B1x2FesSOp1
+  XN9hIWkJVLidnZWfXVd5WB96QYAZb4VuiD9HIgAuxhas0yW34J48sUSyDPqC
+  9McA+lr+2S9BqwPbIc/befDC4RIEAYPDrO2D/Mlg6Xs1Ndf58zmbhGMQDcS5
+  OHRGouLux5WOuoKlzoq2gxmow9LVB97IGi3VSEBROEgYyNwT99tXt7RK8mD+
+  pEWxb8iFzgSwdSF7sW0uYXkEwIDRNyAZdBhhThjcCvor04qMJTWNhGBMb0fA
+  Q5j7Xm2XN5GyS0Yye6kIH09hGVV6YdDPJsaGdzZbc4bRD4hCp6LF+Wm75SE/
+  8xF6hYZB7PIJkwfjwKu4WvTbsNRBiYE2ChJ4KasTfYym33AHoshuLyai8B1N
+  hZqiWAFbb6d12Dj4H0ynpvFQAg0fy6/b0qej4r3meRpcNqxE3ZVwJJvhUe39
+  djZMmjpYOEqfZTRn5vFm5JRSNEAjOcZYv4ZUVvYX34Tw1w55wu7mtZmrauYG
+  aRTysUnzlJmYOO7uRBJQGM3IXw7vtOxAc=]
+certbot_sync_server: internal-sto3-test-cs-1.cdn.sunet.se

internal-sto3-test-cache-2.cdn.sunet.se/overlay/root/.ssh/id_certbot_sync_client.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKa8qWdZCU7Gp4aOfhHlfER4qiCTseCmCc+d2KSqiYGa root@internal-sto3-test-cache-2.cdn.sunet.se

internal-sto3-test-cs-1.cdn.sunet.se/README

@@ -0,0 +1,3 @@
+
+The system documentation is in the docs directory of the multiverse repository.
+

internal-sto3-test-cs-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml

@@ -0,0 +1,8 @@
+---
+certbot_acmed_clients:
+  cdn-test-backend.cdn.sunet.se:
+    allowfrom: []
+    fulldomain: e1964cbb-2f19-41ce-b989-15714a745810.acme-d.sunet.se
+    password: ENC[PKCS7,MIIDCAYJKoZIhvcNAQcDoIIC+TCCAvUCAQAxggKQMIICjAIBADB0MFwxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxLTArBgNVBAMMJGludGVybmFsLXN0bzMtdGVzdC1jcy0xLmNkbi5zdW5ldC5zZQIUJlvw/7wY1TOzXi6H8ObMJGmJvZgwDQYJKoZIhvcNAQEBBQAEggIAYl467khNDapRl/e6mNxm0fssTMqWAMBou3ggthEN1ELsaSgdCUKs5jG6pMHhctrdEu99mG9FqPg3P2NL+8zYVfD+gnu0eysPVANKPT2XZDhTpScQXH0An42cPCZ/zpK12TTvSyeOG6I9iKtYIALDNREtd/Byaw2xkH579XtYVDIxTbJrJHJSvoXOAElCdcDJ7iuC1hF2Ixz+M6Z6ePs9jFyE7Ju6SbrLq7pQY8fFVhAA5cp9/s/7rQPFL9ESzgPKNxOnjIfOclvPzIuPjA8y66aKPKMkPRjlTkH894MjuMGmAF+Db3SKF+fGWz8yFsgZit4fIMG99zSwUT4GJWbmOtMEvNxOQC7oLqK3Epr41m29Vt+dYjsRvQF8rrd5eoTpMuXwYoaCGVwvJsL+ELCPn29jBBFffqrLsRsr2f1zJlVAD9A/YdN0KLbyIFFRzCrQTztOGT5l/0zEilQ2Ru3DdcXHVWD7Sb8Cx8IH76wtgENS6las7smn8zSSIYzrJhjEr78eDpeppqqmKEQ9+QcQ4V7bVvDAfp3e0+tIHlDKMBSm/KrPnGFCnheh5Piy1F0USOxnTYUiHLJXVBXdgRFSQXnwlJvCs6JAX4nd/rpcuzK+fQaVUEu/34ikEzllZ9MU88b77c0hK14McolL0KyDFlJQCmhdk6RsNacIf0tF1oIwXAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQkE0ZZCYx797tDYM8tluLoIAwHPV4+SiGJyf6p9FRUyLLNyqZL5TDu5qqGEL+iCaE6Hy/J80zpCMuYkkB5zzh8BkX]
+    subdomain: e1964cbb-2f19-41ce-b989-15714a745810
+    username: b4d1ebe7-98d4-440d-aa3a-2c958efbdbb8

internal-sto3-test-cs-1.cdn.sunet.se/overlay/etc/hiera/data/local.yaml

@@ -0,0 +1,22 @@
+certbot_sync_client_ssh_keys_db:
+   'internal-sto3-test-cache-1.cdn.sunet.se':
+      key  : 'AAAAC3NzaC1lZDI1NTE5AAAAICwQG6Z0+xcV2HribPifp0MXXY9GLdISS9dHF1+qUdlw'
+      type : 'ssh-ed25519'
+      name : 'internal-sto3-test-cache-1.cdn.sunet.se'
+      options : 'command="/usr/bin/rrsync -no-lock /etc/letsencrypt/",no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding'
+   'internal-sto3-test-cache-2.cdn.sunet.se':
+      key  : 'AAAAC3NzaC1lZDI1NTE5AAAAIKa8qWdZCU7Gp4aOfhHlfER4qiCTseCmCc+d2KSqiYGa'
+      type : 'ssh-ed25519'
+      name : 'internal-sto3-test-cache-2.cdn.sunet.se'
+      options : 'command="/usr/bin/rrsync -no-lock /etc/letsencrypt/",no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding'
+
+certbot_sync_client_ssh_keys_mapping:
+  'root':
+    - 'internal-sto3-test-cache-1.cdn.sunet.se'
+    - 'internal-sto3-test-cache-2.cdn.sunet.se'
+
+certbot_sync_client_ips:
+  - 89.45.237.149 # internal-sto3-test-cache-1.cdn.sunet.se
+  - 2001:6b0:40::4c6 # internal-sto3-test-cache-1.cdn.sunet.se
+  - 192.36.171.94 # internal-sto3-test-cache-2.cdn.sunet.se
+  - 2001:6b0:8:7:526b:8dff:fe80:2fb5 # internal-sto3-test-cache-2.cdn.sunet.se

internal-sto3-test-db-1.cdn.sunet.se/README

@@ -0,0 +1,3 @@
+
+The system documentation is in the docs directory of the multiverse repository.
+

internal-sto3-test-db-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml

@@ -0,0 +1,4 @@
+---
+cdn::db-secrets:
+  postgres_password: ENC[PKCS7,MIIDCAYJKoZIhvcNAQcDoIIC+TCCAvUCAQAxggKQMIICjAIBADB0MFwxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxLTArBgNVBAMMJGludGVybmFsLXN0bzMtdGVzdC1kYi0xLmNkbi5zdW5ldC5zZQIUbbUXduFvDLw3OUVWiGrIvFBkkJMwDQYJKoZIhvcNAQEBBQAEggIAGKLk12OT5zsVKd04qsLkFtawdauLYUERXUC3d9FtZNVpwCFDNMnSsruUfasOvyvdaRbm8AUk/nAGuBNjD9HJj8J45KfQUEAPstnZPHndkF51LwU1twFrZvcSnvFANvxh61MzccMz6NVQL5CXsw4IWMDNhUbkhO5cRfxc0SOVugeTZ74BWwpEww9uKVPtfPRKCgJayBq1o/fyQblGsjJbmu/dRCm32gcdZu1lqfDU0DLsnjk14GJpqpP5h6sEfSrdXyFcWzFzdjtLZLL6TfUWYNYX6CnjjRMv1zZ73877DPXt+vvi0Nvqld5CDTXM9ggDWwZKvluVGn7sTyZdwtWLvs1qK4nui7NLfENtBrUi/GOWsxoFa9tmfeeX/cticzzQcUdDNkfaDgmBa/C7lyjlkwyDGvhYdBHycSEJJ8rxncjBGKl79mpWlK0YTsppgD5eXWZSK7gC3PecRqQ7Jri3aBAWymlM7wfJYP8Z5ocEEq7+BLKSk0Z+Npj8PqJkQ0mw96QC5kNY2LfnGTPbBPpyBZRfZh3F2x1fLN+JQN+IgMpzLW2Oce5HWlPn9iTgCiZU/7mRDJKe8wI4gSg+Mf6hWqlC+NIcMOdAcrfvobzx3PVDHletTd0xgdgGEJpOvtpkYCqcdDbFX4kyEntMaMmp32XBcgsUy1b09uHMB2klzncwXAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQereGW4ZbKx3BV6f/PqkgSYAwcqDeq645K3JJOx9su/j9qDcAGxJN1CqIJjkYFBI+/2euykTaCvIqUMhljoDjeJeE]
+  cdn_password: ENC[PKCS7,MIIDCAYJKoZIhvcNAQcDoIIC+TCCAvUCAQAxggKQMIICjAIBADB0MFwxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxLTArBgNVBAMMJGludGVybmFsLXN0bzMtdGVzdC1kYi0xLmNkbi5zdW5ldC5zZQIUbbUXduFvDLw3OUVWiGrIvFBkkJMwDQYJKoZIhvcNAQEBBQAEggIAC2YvfQOuluAonjkj0iM5DABoSNLXLyjlXfkltBOtAWzlFAQfQNlKd9cArL0qthcge+4AYun9edbyrmKjBAqVYIjPZXMaUjN9HXa07vBwUaHUXUP/rSxL6JYWKAvZTUCnSS+rb/nUM8BAAodk2xNnDrd0H/VN2oBQMkFvWJbCX2/NS9zejr4BpcGTTLjr1GXOuRMwORXwNTHVYZBbZzltnXMRClcdUe9oeIfC2W0BJDTlvAsqVN4DAz985hP8b5vch3uTd59Qzr7pIqpdno8hoI75zdVZ+xeH31rYw5/wqHmsvQK3gvVTtp5zmO4lSWwhiyGfICsX1w/8Fa8j/qR17dfqzkWVWJVN4DN2O/iT1muMtP4WP4j5xvWusE1viW4qGnoFlheo9YVTCP08FLMn0BMyO/7r1AZNG0oDNhTw/DcGYRc0q1bF8L6LjprJrl8Ou7fZbxWcIuQ8UMC+aJPOI0vingTip7/nKhGIBSiplxxKPH3jB2G7NXnimi4sxgAXsXwSSHGTMZH6q46Kc7YrtzT6Vur0W8onQgqFw6Hg5kgybyrdU68skxqAHIQEV3bZ68e5f3MyKY5HxSse8IIngAQdF6mOvOf6JB3zQ98m/JXDDV9FvaMLXSu4iUMGmoHJDO6xmMLaPUamJobM7SFA+0gPa3hAV7fejDdzhh6bLFYwXAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQCT1soHO5e0vaqTWVkkyhS4Awdqh/mowa1LW46di/aaHZp0wPie3eEZnaDsKrIAo4yMFi/Sd0QCXF8YjicLFt0vK5]