Use correct output filename

Also teach sunet-l4lb-namespace to load the nft ruleset if it exists.
While here modify the script so instead of running "once per netns
config file" we merge the interface config from each json file into the
same dict per namespace. Without this we would attempt to load the nft
ruleset twice (once per file that mentioned the namespace) or warn twice
if the file did not exist etc.

edit-secrets

@@ -67,6 +67,39 @@ function patch_broken_eyaml {
                next if @@plugins.include? spec
 
                dependency = spec.dependencies.find { |d| d.name == "hiera-eyaml" }
+EOF
+            fi
+        fi
+    fi
+
+    #
+    # Ubuntu 24.04 (noble) has a hiera-eyaml version that is incompatible with ruby 3.2+ (default in ubuntu24).
+    # This is fixed in hiera-eyaml version 3.3.0: https://github.com/voxpupuli/hiera-eyaml/pull/340/files
+    # https://github.com/voxpupuli/hiera-eyaml/blob/master/CHANGELOG.md
+    # But there is no modern version of hiera-eyaml packaged in debian or ubuntu.
+    # https://github.com/puppetlabs/puppet/wiki/Puppet-8-Compatibility#filedirexists-removed
+    #
+
+    . /etc/os-release
+    if [ "${VERSION_CODENAME}" == "noble" ]; then
+        plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb"
+        if [ -f $plugins_file ]; then
+            # We only want to try patching the file if it is the known broken version
+            bad_sum="59c6eb910ab2eb44f8c75aeaa79bff097038feb673b5c6bdccde23d9b2a393e2"
+            sum=$(sha256sum $plugins_file | awk '{print $1}')
+            if [ "$sum" == "$bad_sum" ]; then
+                patch --fuzz=0 --directory=/ --strip=0 <<'EOF'
+--- /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb.orig	2022-06-11 16:30:10.000000000 +0000
++++ /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb	2024-09-09 14:13:19.306342025 +0000
+@@ -59,7 +59,7 @@
+             Optimist::die "You must specify an eyaml file" if ARGV.empty?
+             options[:source] = :eyaml
+             options[:eyaml] = ARGV.shift
+-            if File.exists? options[:eyaml]
++            if File.exist? options[:eyaml]
+               begin
+                 options[:input_data] = File.read options[:eyaml]
+               rescue
 EOF
             fi
         fi

global/overlay/etc/puppet/cosmos-rules.yaml

@@ -19,10 +19,11 @@
 '^internal-.+-test-mqtt-[0-9]+\.cdn\.sunet\.se$':
   cdn::ca_trust:
   cdn::mqtt:
-    dc: tug
+    dc: sto3
     clients:
-      - shared-tug-test-cache-1.cdn.sunet.se
-      - shared-tug-test-cache-2.cdn.sunet.se
+      - internal-sto3-test-cache-2.cdn.sunet.se
+    client_ips:
+      - 192.36.171.94 # internal-sto3-test-cache-2.cdn.sunet.se
 
 '^internal-.+-test-ca-[0-9]+\.cdn\.sunet\.se$':
   sunet::dockerhost2:
@@ -36,3 +37,7 @@
 '^internal-.+-test-cs-[0-9]+\.cdn\.sunet\.se$':
   sunet::certbot::acmed:
   sunet::certbot::sync::server:
+
+'^internal-.+-test-db-[0-9]+\.cdn\.sunet\.se$':
+  sunet::dockerhost2:
+  cdn::db:

global/overlay/etc/puppet/modules/cdn/files/ca/configure-acme

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# When initializing step-ca with the docker flag DOCKER_STEPCA_INIT_ACME
+# a basic ACME provisioner is enabled. This script runs commands to modify the
+# default configuration.
+
+# Enable forceCN if not set.
+# This is needed because certbot does not include a
+# Subject CN field in the CSR:
+# https://github.com/certbot/certbot/issues/9633#issuecomment-1484988078
+# ... and the Mosquitto MQTT server uses the Subject CN in ACL filters.
+#
+# Ideally Mosquitto would learn to look at the SAN field instead:
+# https://github.com/eclipse-mosquitto/mosquitto/issues/2511
+if [ "$(step ca provisioner list | jq -r '.[] | select (.name == "acme") | .forceCN')" = "null" ]; then
+    step ca provisioner update acme --force-cn --admin-subject=step --admin-provisioner=admin  --admin-password-file=/opt/step-ca/init/secrets/provisioner-password
+fi

global/overlay/etc/puppet/modules/cdn/files/ca/set-provisioner-pw

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-# When initializing step-ca with the docker flag STEPCA_INIT_PASSWORD_FILE the
-# password will be used both for key encryption as well as the admin "step"
-# provisioner. If not using that flag a separate password will be generated for
-# each usage. This seems better as you are not typing the encryption password
-# any other time, while the provisioner password is used anytime you are
-# managing things.
+# When initializing step-ca with the docker flag
+# DOCKER_STEPCA_INIT_PASSWORD_FILE the password will be used both for key
+# encryption as well as the admin "step" provisioner. If not using that flag a
+# separate password will be generated for each usage. This seems better as you
+# are not typing the encryption password any other time, while the provisioner
+# password is used anytime you are managing things.
 #
 # This script is used on first setup of step-ca to modify the provisioner to
 # use its own password instead. Pending

global/overlay/etc/puppet/modules/cdn/files/db/init-cdn-db.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+# shellcheck source=/dev/null
+. /conf/init-cdn-db.conf
+
+# Create database named after user, then create a schema named the same as the
+# user which is also owned by that user. Because search_path (SHOW
+# search_path;) starts with "$user" by default this means any tables will be
+# created in that user-specific SCHEMA by default instead of falling back to
+# "public". This follows the "secure schema usage pattern" summarized as
+# "Constrain ordinary users to user-private schemas" from
+# https://www.postgresql.org/docs/current/ddl-schemas.html#DDL-SCHEMAS-PATTERNS
+#
+# "In PostgreSQL 15 and later, the default configuration supports this usage
+# pattern. In prior versions, or when using a database that has been upgraded
+# from a prior version, you will need to remove the public CREATE privilege
+# from the public schema"
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+	CREATE USER cdn WITH PASSWORD '${cdn_password:?}';
+	CREATE DATABASE cdn;
+	GRANT ALL PRIVILEGES ON DATABASE cdn TO cdn;
+	\c cdn;
+	CREATE SCHEMA cdn AUTHORIZATION cdn;
+EOSQL

global/overlay/etc/puppet/modules/cdn/files/l4lb/sunet-l4lb-namespace

@@ -13,7 +13,9 @@ pylint sunet-l4lb-namespace
 mypy --strict sunet-l4lb-namespace
 """
 
+import ipaddress
 import json
+import os
 import shlex
 import subprocess
 import sys
@@ -21,6 +23,7 @@ import sys
 
 def run_command(cmd: str) -> subprocess.CompletedProcess[str]:
     """Execute subprocess command"""
+    print(f"{cmd}")
     args = shlex.split(cmd)
     try:
         proc = subprocess.run(args, capture_output=True, check=True, encoding="utf-8")
@@ -35,52 +38,95 @@ def run_command(cmd: str) -> subprocess.CompletedProcess[str]:
     return proc
 
 
-def configure_interfaces(
+def configure_interfaces(  # pylint: disable=too-many-locals,too-many-branches
     namespace: str, if_data: dict[str, dict[str, list[str]]]
 ) -> None:
     """Configure interfaces"""
-    proc = run_command("ip netns exec l4lb ip -j addr show")
+    proc = run_command(f"ip netns exec {namespace} ip -j addr show")
     namespace_ifs = json.loads(proc.stdout)
+
+    ipv4_key = "ipv4"
+    ipv6_key = "ipv6"
     for if_name, data in if_data.items():
         if_exists = next(
             (True for interface in namespace_ifs if interface["ifname"] == if_name),
             False,
         )
         if not if_exists:
-            run_command(f"ip link set {if_name} netns {namespace}")
+            if if_name.startswith("dummy"):
+                run_command(
+                    f"ip netns exec {namespace} ip link add {if_name} type dummy"
+                )
+            else:
+                run_command(f"ip link set {if_name} netns {namespace}")
+
+            run_command(f"ip netns exec {namespace} ip link set {if_name} up")
 
         proc = run_command(f"ip netns exec {namespace} ip -j addr show dev {if_name}")
         if_conf = json.loads(proc.stdout)
-        for ipv4_cidr in data["ipv4"]:
-            ip4, prefix = ipv4_cidr.split("/")
-            v4_addr_exists = next(
-                (
-                    True
-                    for addr in if_conf[0]["addr_info"]
-                    if addr["local"] == ip4 and addr["prefixlen"] == int(prefix)
-                ),
-                False,
-            )
-            if not v4_addr_exists:
-                run_command(
-                    f"ip netns exec {namespace} ip addr add {ipv4_cidr} dev {if_name}"
+
+        # Add missing addresses from config
+        if ipv4_key in data:
+            for configured_ipv4_cidr in data[ipv4_key]:
+                ip4, prefix = configured_ipv4_cidr.split("/")
+                v4_addr_exists = next(
+                    (
+                        True
+                        for addr in if_conf[0]["addr_info"]
+                        if addr["local"] == ip4 and addr["prefixlen"] == int(prefix)
+                    ),
+                    False,
                 )
-        for ipv6_cidr in data["ipv6"]:
-            ip6, prefix = ipv6_cidr.split("/")
-            v6_addr_exists = next(
-                (
-                    True
-                    for addr in if_conf[0]["addr_info"]
-                    if addr["local"] == ip6 and addr["prefixlen"] == int(prefix)
-                ),
-                False,
-            )
-            if not v6_addr_exists:
-                run_command(
-                    f"ip netns exec {namespace} ip addr add {ipv6_cidr} dev {if_name}"
+                if not v4_addr_exists:
+                    run_command(
+                        f"ip netns exec {namespace} ip addr add {configured_ipv4_cidr} dev {if_name}"  # pylint: disable=line-too-long
+                    )
+        if ipv6_key in data:
+            for ipv6_cidr in data[ipv6_key]:
+                ip6, prefix = ipv6_cidr.split("/")
+                v6_addr_exists = next(
+                    (
+                        True
+                        for addr in if_conf[0]["addr_info"]
+                        if addr["local"] == ip6 and addr["prefixlen"] == int(prefix)
+                    ),
+                    False,
+                )
+                if not v6_addr_exists:
+                    run_command(
+                        f"ip netns exec {namespace} ip addr add {ipv6_cidr} dev {if_name}"
+                    )
+
+        # Remove no longer configured addresseses
+        for addr_info in if_conf[0]["addr_info"]:
+            # Ignore addresses like fe80
+            if addr_info["scope"] != "global":
+                continue
+
+            cidr = "/".join((addr_info["local"], str(addr_info["prefixlen"])))
+
+            # We need strict=False because otherwise ip_network() gets angry if
+            # there are host bits set in the address (which of course there is
+            # because we are parsing actual interface configs, not pure
+            # "networks")
+            cidr_net = ipaddress.ip_network(cidr, strict=False)
+
+            needs_removal = False
+            if cidr_net.version == 4:
+                if ipv4_key not in data or cidr not in data[ipv4_key]:
+                    needs_removal = True
+            elif cidr_net.version == 6:
+                if ipv6_key not in data or cidr not in data[ipv6_key]:
+                    needs_removal = True
+            else:
+                raise ValueError(
+                    f"Expected IPv4 or IPv6, got something else: {cidr_net.version}"
                 )
 
-        run_command(f"ip netns exec {namespace} ip link set {if_name} up")
+            if needs_removal:
+                run_command(
+                    f"ip netns exec {namespace} ip addr del {cidr} dev {if_name}"
+                )
 
 
 def setup_namespaces(netns_data: dict[str, dict[str, dict[str, list[str]]]]) -> None:
@@ -94,8 +140,17 @@ def setup_namespaces(netns_data: dict[str, dict[str, dict[str, list[str]]]]) ->
         if not netns_exists:
             run_command(f"ip netns add {namespace}")
 
-        # Make localhost available
-        run_command(f"ip netns exec {namespace} ip link set lo up")
+            # Make localhost available
+            run_command(f"ip netns exec {namespace} ip link set lo up")
+
+        # (Re)load the nft ruleset for the given namespace
+        nft_ruleset = f"/opt/sunet-cdn/l4lb/conf/nft-{namespace}.conf"
+        if os.path.isfile(nft_ruleset):
+            run_command(f"ip netns exec {namespace} nft -f {nft_ruleset}")
+        else:
+            print(
+                f"WARNING: no nft ruleset found for namespace '{namespace}' ({nft_ruleset}), the namespace will not be firewalled"  # pylint: disable=line-too-long
+            )
 
         configure_interfaces(namespace, if_data)
 
@@ -124,10 +179,30 @@ def main() -> None:
     #    }
     #  }
     # }
-    with open("/opt/sunet-cdn/l4lb/conf/netns.json", encoding="utf-8") as f:
-        netns_data = json.load(f)
 
-    setup_namespaces(netns_data)
+    input_files = [
+        "/opt/sunet-cdn/l4lb/conf/netns-base.json",
+        "/opt/sunet-cdn/l4lb/conf/netns-sunet-cdn-agent.json",
+    ]
+
+    merged_netns_data: dict[str, dict[str, dict[str, list[str]]]] = {}
+    for input_file in input_files:
+        try:
+            with open(input_file, encoding="utf-8") as f:
+                netns_data = json.load(f)
+
+            # Combine interface config from multiple files belonging to the same namespace
+            for ns, ns_data in netns_data.items():
+                if ns in merged_netns_data:
+                    merged_netns_data[ns].update(ns_data)
+                else:
+                    merged_netns_data[ns] = ns_data
+
+        except FileNotFoundError:
+            print(f"skipping nonexistant file '{input_file}'")
+            continue
+
+    setup_namespaces(merged_netns_data)
 
 
 if __name__ == "__main__":

global/overlay/etc/puppet/modules/cdn/files/mqtt/sunet-cdn-mqtt

@@ -9,20 +9,13 @@ set -eu
 le_dir="/etc/letsencrypt/live/$(hostname -f)"
 mosquitto_dir="/etc/mosquitto"
 
-le_chain="$le_dir/chain.pem"
-mosquitto_chain="$mosquitto_dir/ca_certificates/chain.pem"
-cp "$le_chain" "$mosquitto_chain"
-chown mosquitto:root "$mosquitto_chain"
-
-le_cert="$le_dir/cert.pem"
-mosquitto_cert="$mosquitto_dir/certs/cert.pem"
-cp "$le_cert" "$mosquitto_cert"
-chown mosquitto:root "$mosquitto_cert"
+le_fullchain="$le_dir/fullchain.pem"
+mosquitto_fullchain="$mosquitto_dir/certs/fullchain.pem"
+install -m 644 -o mosquitto -g root "$le_fullchain" "$mosquitto_fullchain"
 
 le_key="$le_dir/privkey.pem"
 mosquitto_key="$mosquitto_dir/certs/privkey.pem"
-cp "$le_key" "$mosquitto_key"
-chown mosquitto:root "$mosquitto_key"
+install -m 600 -o mosquitto -g root "$le_key" "$mosquitto_key"
 
 # Tell mosquitto to reload certs
 pkill -x -HUP mosquitto

global/overlay/etc/puppet/modules/cdn/manifests/ca.pp

@@ -59,6 +59,14 @@ class cdn::ca(
     content => file('cdn/ca/bootstrap-client'),
   }
 
+  file { '/opt/step-ca/init/scripts/configure-acme':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => file('cdn/ca/configure-acme'),
+  }
+
   file { '/opt/step-ca/init/deb':
     ensure => directory,
     owner  => 'root',

global/overlay/etc/puppet/modules/cdn/manifests/cache.pp

@@ -2,9 +2,19 @@
 class cdn::cache(
   Hash[String, Integer] $customers = {
     customer1 => 1000000000,
-  }
+  },
+  String $sunet_cdnp_version = '0.0.7',
+  Hash[String, String] $acme_url = {
+    test => 'https://internal-sto3-test-ca-1.cdn.sunet.se:9000/acme/acme/directory'
+  },
+  Hash[String, Hash[String, String]] $mqtt_url = {
+    sto3 => {
+      test => 'tls://internal-sto3-test-mqtt-1.cdn.sunet.se:8883',
+    },
+  },
 )
 {
+  include sunet::systemd_reload
   include sunet::packages::certbot
   include cdn::ca_trust
 
@@ -71,6 +81,22 @@ class cdn::cache(
     content => template('cdn/cache/10-cdn-ipip.network.erb'),
   }
 
+  file { '/etc/systemd/network/10-cdn-ip6tunl.netdev.erb':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-ip6tunl.netdev.erb'),
+  }
+
+  file { '/etc/systemd/network/10-cdn-ip6tunl.network.erb':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/10-cdn-ip6tunl.network.erb'),
+  }
+
   # Reload the network config if it has changed
   exec { 'networkctl reload':
     subscribe   => [File['/etc/systemd/network/10-cdn-dummy.network'], File['/etc/systemd/network/10-cdn-ipip.network']],
@@ -91,16 +117,92 @@ class cdn::cache(
     refreshonly => true,
   }
 
-  # Allow tunnel packets arriving from l4lb nodes
+  # Allow IPv4 tunnel packets arriving from l4lb nodes
   sunet::nftables::rule { 'sunet_cdn_tunnel4':
     rule => 'add rule inet filter input ip saddr { 130.242.64.233, 130.242.64.235 } ip protocol ipencap counter accept comment "sunet-cdn-tunnel4"'
   }
 
+  # Allow IPv6 tunnel packets arriving from l4lb nodes
+  sunet::nftables::rule { 'sunet_cdn_tunnel6':
+    rule => 'add rule inet filter input ip6 saddr { 2001:6b0:2006:74::1, 2001:6b0:2006:75::1 } ip6 nexthdr ipv6 counter accept comment "sunet-cdn-tunnel6"'
+  }
+
   # Allow decapsulated tunnel packets targeting the service IP range to reach
   # local service ports
   sunet::nftables::rule { 'sunet_cdn_service4':
     rule => 'add rule inet filter input meta iifname tunl0 ip daddr 188.240.152.0/24 tcp dport { 80, 443 } counter accept comment "sunet-cdn-service4"'
   }
+  sunet::nftables::rule { 'sunet_cdn_service6':
+    rule => 'add rule inet filter input meta iifname ip6tnl0 ip6 daddr 2001:6b0:2100::/48 tcp dport { 80, 443 } counter accept comment "sunet-cdn-service6"'
+  }
+
+  # From https://wiki.sunet.se/display/sunetops/Platform+naming+standards
+  $my_fqdn = $facts['networking']['fqdn']
+  $dot_split = split($my_fqdn, '[.]')
+  $my_hostname = $dot_split[0]
+  $dash_split = split($my_hostname,'[-]')
+  $location = $dash_split[1]
+  $environment = $dash_split[2]
+
+  sunet::nftables::allow { 'allow-step-ca-acme':
+    from  => '89.45.237.248', # internal-sto3-test-ca-1.cdn.sunet.se
+    port  => 80,
+    proto => 'tcp',
+  }
+
+  # Get client cert for connecting to MQTT bus
+  exec { "certbot certonly -n --email patlu@sunet.se --no-eff-email --agree-tos --standalone -d ${my_fqdn} --server ${acme_url[$environment]} --http-01-address ${facts['networking']['ip']}":
+    creates => "/etc/letsencrypt/live/${my_fqdn}/fullchain.pem"
+  }
+
+  $sunet_cdnp_dir = '/var/lib/sunet-cdnp'
+  $sunet_cdnp_file = "sunet-cdnp_${sunet_cdnp_version}_linux_${facts[os][architecture]}.tar.gz"
+  $sunet_cdnp_url = "https://github.com/SUNET/sunet-cdnp/releases/download/v${sunet_cdnp_version}/${sunet_cdnp_file}"
+  # Create directory for managing CDP purger
+  file { $sunet_cdnp_dir:
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  exec { "curl -LO ${sunet_cdnp_url}":
+    creates => "${sunet_cdnp_dir}/${sunet_cdnp_file}",
+    cwd     => $sunet_cdnp_dir,
+    notify  => Exec['extract sunet-cdnp'],
+  }
+
+  exec { 'extract sunet-cdnp':
+    command     => "tar -xzf ${sunet_cdnp_file} sunet-cdnp",
+    cwd         => $sunet_cdnp_dir,
+    refreshonly => true,
+    notify      => Service['sunet-cdnp'],
+  }
+
+  file { "${sunet_cdnp_dir}/sunet-cdnp":
+    owner => 'root',
+    group => 'root',
+    mode  => '0755',
+  }
+
+  file { '/usr/local/bin/sunet-cdnp':
+    ensure => link,
+    target => "${sunet_cdnp_dir}/sunet-cdnp",
+  }
+
+  file { '/etc/systemd/system/sunet-cdnp.service':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/cache/sunet-cdnp.service.erb'),
+    notify  => [Class['sunet::systemd_reload']],
+  }
+
+  service { 'sunet-cdnp':
+    ensure => 'running',
+    enable => true,
+  }
 
   if $cache_secrets {
     $customers.each |String $customer, Integer $customer_uid| {
@@ -151,13 +253,13 @@ class cdn::cache(
 
         concat::fragment { "${customer}-fullchain-${cache_secrets['customers'][$customer]['host']}":
           target => $combined_pem,
-          source => "/etc/letsencrypt/live/${cache_secrets['customers'][$customer]['host']}/fullchain.pem",
+          source => "/opt/certbot-sync/letsencrypt/live/${cache_secrets['customers'][$customer]['host']}/fullchain.pem",
           order  => '01',
         }
 
         concat::fragment { "${customer}-privkey-${cache_secrets['customers'][$customer]['host']}":
           target => $combined_pem,
-          source => "/etc/letsencrypt/live/${cache_secrets['customers'][$customer]['host']}/privkey.pem",
+          source => "/opt/certbot-sync/letsencrypt/live/${cache_secrets['customers'][$customer]['host']}/privkey.pem",
           order  => '02',
         }
 

global/overlay/etc/puppet/modules/cdn/manifests/db.pp

@@ -0,0 +1,76 @@
+# Configure a SUNET CDN DB server
+class cdn::db(
+  String $postgres_version = '17.0-bookworm',
+)
+{
+
+  $db_secrets = lookup({ 'name' => 'cdn::db-secrets', 'default_value' => undef })
+
+  if $db_secrets {
+    file { '/opt/sunet-cdn':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0755',
+    }
+
+    file { '/opt/sunet-cdn/compose':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0750',
+    }
+
+    file { '/opt/sunet-cdn/db':
+      ensure => directory,
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0750',
+    }
+
+    # User/group 999 matches postgres user in container
+    file { '/opt/sunet-cdn/db/conf':
+      ensure => directory,
+      owner  => '999',
+      group  => '999',
+      mode   => '0750',
+    }
+
+    file { '/opt/sunet-cdn/db/docker-entrypoint-initdb.d':
+      ensure => directory,
+      owner  => '999',
+      group  => '999',
+      mode   => '0750',
+    }
+
+    file { '/opt/sunet-cdn/db/conf/init-cdn-db.conf':
+      ensure => file,
+      owner  => '999',
+      group  => '999',
+      mode   => '0640',
+      content => template('cdn/db/init-cdn-db.conf.erb'),
+    }
+
+    file { '/opt/sunet-cdn/db/docker-entrypoint-initdb.d/init-cdn-db.sh':
+      ensure => file,
+      owner  => '999',
+      group  => '999',
+      mode   => '0750',
+      content => file('cdn/db/init-cdn-db.sh'),
+    }
+
+    sunet::nftables::docker_expose { 'postgres-db' :
+      allow_clients => '127.0.0.1',
+      port          => 5432,
+      iif           => $facts['networking']['primary'],
+    }
+
+    sunet::docker_compose { 'sunet-cdn-db':
+      content          => template('cdn/db/docker-compose.yml.erb'),
+      service_name     => 'cdn-db',
+      compose_dir      => '/opt/sunet-cdn/compose',
+      compose_filename => 'docker-compose.yml',
+      description      => 'SUNET CDN DB',
+    }
+  }
+}

global/overlay/etc/puppet/modules/cdn/manifests/l4lb.pp

@@ -15,33 +15,61 @@ class cdn::l4lb(
     }
   }
 
+  include sunet::systemd_reload
+
+  package {'conntrack': ensure => installed }
+
+  package {'bird2': ensure => installed }
+
   file { '/opt/sunet-cdn':
-    ensure  => directory,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0640',
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0640',
   }
 
+  $sysctl_file = '/etc/sysctl.d/99-cdn-l4lb.conf'
+    file { $sysctl_file:
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('cdn/l4lb/sysctl.erb'),
+    }
+    # Load the sysctl file if it has changed
+    exec { "sysctl -p ${sysctl_file}":
+      subscribe   => File[$sysctl_file],
+      refreshonly => true,
+    }
+
   file { '/opt/sunet-cdn/l4lb':
-    ensure  => directory,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0640',
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0640',
   }
 
   file { '/opt/sunet-cdn/l4lb/conf':
-    ensure  => directory,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0640',
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0640',
   }
 
-  file { '/opt/sunet-cdn/l4lb/conf/netns.json':
+  file { '/opt/sunet-cdn/l4lb/conf/netns-base.json':
     ensure  => file,
     owner   => 'root',
     group   => 'root',
     mode    => '0644',
-    content => template('cdn/l4lb/netns.json.erb'),
+    content => template('cdn/l4lb/netns-base.json.erb'),
+  }
+
+  file { '/opt/sunet-cdn/l4lb/conf/nft-l4lb.conf':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/l4lb/nft-l4lb.conf.erb'),
   }
 
   file { '/usr/local/bin/sunet-l4lb-namespace':
@@ -59,4 +87,34 @@ class cdn::l4lb(
     mode    => '0644',
     content => template('cdn/l4lb/sunet-l4lb-namespace.service.erb'),
   }
+
+  file { '/etc/systemd/system/bird.service.d':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  file { '/etc/systemd/system/bird.service.d/override.conf':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('cdn/l4lb/bird-override.conf.erb'),
+    notify  => [Class['sunet::systemd_reload']]
+  }
+
+  file { '/etc/bird/bird.conf':
+    ensure  => file,
+    owner   => 'bird',
+    group   => 'bird',
+    mode    => '0640',
+    content => template('cdn/l4lb/bird.conf.erb'),
+    notify  => Service['bird'],
+  }
+
+  service { 'bird':
+    ensure => 'running',
+    enable => true,
+  }
 }

global/overlay/etc/puppet/modules/cdn/manifests/mqtt.pp

@@ -2,6 +2,7 @@
 class cdn::mqtt(
   String $dc = '',
   Array[String] $clients = [],
+  Array[String] $client_ips = [],
   Hash[String, Hash] $bridges = {},
   Hash[String, String] $acme_url = {
     test => 'https://internal-sto3-test-ca-1.cdn.sunet.se:9000/acme/acme/directory'
@@ -35,6 +36,14 @@ class cdn::mqtt(
     proto => 'tcp',
   }
 
+  $client_ips.each | String $client_ip | {
+    sunet::nftables::allow { "allow-acme-client-${client_ip}":
+      from  => $client_ip,
+      port  => 8883,
+      proto => 'tcp',
+    }
+  }
+
   # From https://wiki.sunet.se/display/sunetops/Platform+naming+standards
   $my_fqdn = $facts['networking']['fqdn']
   $dot_split = split($my_fqdn, '[.]')

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-ip6tunl.netdev.erb

@@ -0,0 +1,16 @@
+[NetDev]
+Name=ip6tnl0
+Kind=ip6tnl
+
+[Tunnel]
+Independent=true
+# Local and Remote should be 'any' or '::' but right now it does not work.
+# However, by using the name "ip6tnl0" that matches the name of the interface
+# automatically created when the kernel module is loaded it seems we can set
+# anything "non-zero" here and the config will be accepted but still leaving
+# "::" as both Local and Remote on the interface. This then results in the
+# wanted configuration for now. One bug cancelling out another bug?
+# https://github.com/systemd/systemd/issues/34930
+Local=::1
+Remote=::1
+Mode=ip6ip6

global/overlay/etc/puppet/modules/cdn/templates/cache/10-cdn-ip6tunl.network.erb

@@ -0,0 +1,2 @@
+[Match]
+Name=ip6tnl0

global/overlay/etc/puppet/modules/cdn/templates/cache/sunet-cdnp.service.erb

@@ -0,0 +1,16 @@
+# This service file is generated by Puppet. Do not edit.
+[Unit]
+Description=SUNET CDN Purger
+Wants=docker.service
+After=docker.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/sunet-cdnp \
+  -mqtt-ca-file /usr/local/share/ca-certificates/step_ca_root.crt \
+  -mqtt-client-key-file /etc/letsencrypt/live/<%= @networking['fqdn'] %>/privkey.pem \
+  -mqtt-client-cert-file /etc/letsencrypt/live/<%= @networking['fqdn'] %>/fullchain.pem \
+  -mqtt-server <%= @mqtt_url[@location][@environment] %>
+
+[Install]
+WantedBy=multi-user.target

global/overlay/etc/puppet/modules/cdn/templates/db/docker-compose.yml.erb

@@ -0,0 +1,13 @@
+services:
+  db:
+    image: "postgres:<%= @postgres_version %>"
+    environment:
+      - POSTGRES_PASSWORD=<%= @db_secrets['postgres_password'] %>
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - /opt/sunet-cdn/db/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+      - /opt/sunet-cdn/db/conf:/conf
+volumes:
+  postgres_data:

global/overlay/etc/puppet/modules/cdn/templates/db/init-cdn-db.conf.erb

@@ -0,0 +1,2 @@
+# File sourced by init-cdn-db.sh
+cdn_password="<%= @db_secrets['cdn_password'] %>"

global/overlay/etc/puppet/modules/cdn/templates/l4lb/bird-override.conf.erb

@@ -0,0 +1,4 @@
+[Service]
+NetworkNamespacePath=/var/run/netns/l4lb
+Wants=sunet-l4lb-namespace.service
+After=sunet-l4lb-namespace.service

global/overlay/etc/puppet/modules/cdn/templates/l4lb/bird.conf.erb

@@ -0,0 +1,115 @@
+# Configure logging
+log syslog all;
+
+# Set router ID. It is a unique identification of your router, usually one of
+# IPv4 addresses of the router. It is recommended to configure it explicitly.
+router id <%= @facts['networking']['ip'] %>;
+
+# The Device protocol is not a real routing protocol. It does not generate any
+# routes and it only serves as a module for getting information about network
+# interfaces from the kernel. It is necessary in almost any configuration.
+protocol device {
+}
+
+# The direct protocol is not a real routing protocol. It automatically generates
+# direct routes to all network interfaces. Can exist in as many instances as you
+# wish if you want to populate multiple routing tables with direct routes.
+protocol direct {
+  ipv4;
+  ipv6;
+  interface "dummy0";
+}
+
+# The Kernel protocol is not a real routing protocol. Instead of communicating
+# with other routers in the network, it performs synchronization of BIRD
+# routing tables with the OS kernel. One instance per table.
+protocol kernel {
+  ipv4 {
+    import none;
+    export filter { if source = RTS_DEVICE then { reject; } accept; };
+  };
+# Create ECMP routes in kernel table from multiple paths
+  merge paths;
+}
+
+protocol kernel {
+  ipv6 {
+    import none;
+    export filter { if source = RTS_DEVICE then { reject; } accept; };
+  };
+  # Create ECMP routes in kernel table from multiple paths
+  merge paths;
+}
+
+protocol bgp tug_r11_v4 {
+  description "tug-r11-v4";
+  local 130.242.64.233 as 65443;
+  neighbor 130.242.64.232 as 1653;
+  hold time 90;
+
+  ipv4 {
+    import filter {
+      if net ~ [ 0.0.0.0/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 188.240.152.0/24{32,32} ] then
+        accept; else reject;
+      };
+  };
+}
+
+protocol bgp tug_r11_v6 {
+  description "tug-r12-v6";
+  local 2001:6b0:2006:74::1 as 65443;
+  neighbor 2001:6b0:2006:74:: as 1653;
+  hold time 90;
+
+  ipv6 {
+    import filter {
+      if net ~ [ ::/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 2001:6b0:2100::/48{128,128} ] then
+        accept; else reject;
+    };
+  };
+}
+
+
+protocol bgp tug_r12_v4 {
+  description "tug-r12-v4";
+  local 130.242.64.235 as 65443;
+  neighbor 130.242.64.234 as 1653;
+  hold time 90;
+
+  ipv4 {
+    import filter {
+      if net ~ [ 0.0.0.0/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 188.240.152.0/24{32,32} ] then
+        accept; else reject;
+      };
+  };
+}
+
+protocol bgp tug_r12_v6 {
+  description "tug-r12-v6";
+  local 2001:6b0:2006:75::1 as 65443;
+  neighbor 2001:6b0:2006:75:: as 1653;
+  hold time 90;
+
+  ipv6 {
+    import filter {
+      if net ~ [ ::/0 ] then
+        accept; else reject;
+    };
+    export filter {
+      if net ~ [ 2001:6b0:2100::/48{128,128} ] then
+        accept; else reject;
+    };
+  };
+}

global/overlay/etc/puppet/modules/cdn/templates/l4lb/nft-l4lb.conf.erb

@@ -0,0 +1,40 @@
+#!/usr/sbin/nft -f
+
+flush ruleset
+
+table inet filter {
+        chain input {
+                type filter hook input priority 0; policy drop;
+
+                # accept any localhost traffic
+                iif lo counter accept
+
+                # accept icmp
+                ip protocol icmp counter accept
+                ip6 nexthdr icmpv6 icmpv6 type { destination-unreachable, packet-too-big, time-exceeded,
+                                                 parameter-problem, echo-request, mld-listener-query,
+                                                 nd-router-solicit, nd-router-advert, nd-neighbor-solicit,
+                                                 nd-neighbor-advert } counter accept
+
+                # accept traffic originated from us
+                ct state established counter accept
+                # silently drop invalid packets
+                ct state invalid counter drop
+        }
+        chain forward {
+                type filter hook forward priority 0; policy drop;
+        }
+        chain output {
+                type filter hook output priority 0;
+        }
+}
+
+# HTTP and HTTPS
+add rule inet filter input tcp dport 80 counter accept comment "l4lb HTTP"
+add rule inet filter input tcp dport 443 counter accept comment "l4lb HTTPS"
+
+# BGP
+add rule inet filter input ip saddr { 130.242.64.232 } tcp dport 179 counter accept comment "tug-r11-v4"
+add rule inet filter input ip saddr { 130.242.64.234 } tcp dport 179 counter accept comment "tug-r12-v4"
+add rule inet filter input ip6 saddr { 2001:6b0:2006:74:: } tcp dport 179 counter accept comment "tug-r11-v6"
+add rule inet filter input ip6 saddr { 2001:6b0:2006:75:: } tcp dport 179 counter accept comment "tug-r12-v6"

global/overlay/etc/puppet/modules/cdn/templates/l4lb/sysctl.erb

@@ -0,0 +1,5 @@
+# Allow non-SYN packets to be accepted for a IPVS service.
+# Needed if another l4lb has been taken down and routers are now sending us
+# packets for existing connections that was previously handled by the other
+# node.
+net.ipv4.vs.sloppy_tcp=1

global/overlay/etc/puppet/modules/cdn/templates/mqtt/cdn.conf.erb

@@ -1,6 +1,6 @@
 listener 8883
-cafile /etc/mosquitto/ca_certificates/chain.pem
-certfile /etc/mosquitto/certs/cert.pem
+cafile /usr/local/share/ca-certificates/step_ca_root.crt
+certfile /etc/mosquitto/certs/fullchain.pem
 keyfile /etc/mosquitto/certs/privkey.pem
 require_certificate true
 use_identity_as_username true

global/overlay/etc/puppet/setup_cosmos_modules

@@ -83,7 +83,7 @@ def main():
         "sunet": {
             "repo": "https://github.com/SUNET/puppet-sunet.git",
             "upgrade": "yes",
-            "tag": "stable-2023v1-2*",
+            "tag": "patlu-dockerhost2-ipv6-nat-2*",
         },
         "augeas": {
             "repo": "https://github.com/SUNET/puppet-augeas.git",

global/pre-tasks.d/020common-tools

@@ -8,7 +8,7 @@ set -e
 stamp="$COSMOS_BASE/stamps/common-tools-v01.stamp"
 
 if ! test -f $stamp; then
-    apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades ntp
+    apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades
     update-alternatives --set editor /usr/bin/vim.basic
 
     mkdir -p `dirname $stamp`

internal-sto3-test-cs-1.cdn.sunet.se/overlay/etc/hiera/data/local.yaml

@@ -19,3 +19,4 @@ certbot_sync_client_ips:
   - 89.45.237.149 # internal-sto3-test-cache-1.cdn.sunet.se
   - 2001:6b0:40::4c6 # internal-sto3-test-cache-1.cdn.sunet.se
   - 192.36.171.94 # internal-sto3-test-cache-2.cdn.sunet.se
+  - 2001:6b0:8:7:526b:8dff:fe80:2fb5 # internal-sto3-test-cache-2.cdn.sunet.se

internal-sto3-test-db-1.cdn.sunet.se/README

@@ -0,0 +1,3 @@
+
+The system documentation is in the docs directory of the multiverse repository.
+

internal-sto3-test-db-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml

@@ -0,0 +1,4 @@
+---
+cdn::db-secrets:
+  postgres_password: ENC[PKCS7,MIIDCAYJKoZIhvcNAQcDoIIC+TCCAvUCAQAxggKQMIICjAIBADB0MFwxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxLTArBgNVBAMMJGludGVybmFsLXN0bzMtdGVzdC1kYi0xLmNkbi5zdW5ldC5zZQIUbbUXduFvDLw3OUVWiGrIvFBkkJMwDQYJKoZIhvcNAQEBBQAEggIAGKLk12OT5zsVKd04qsLkFtawdauLYUERXUC3d9FtZNVpwCFDNMnSsruUfasOvyvdaRbm8AUk/nAGuBNjD9HJj8J45KfQUEAPstnZPHndkF51LwU1twFrZvcSnvFANvxh61MzccMz6NVQL5CXsw4IWMDNhUbkhO5cRfxc0SOVugeTZ74BWwpEww9uKVPtfPRKCgJayBq1o/fyQblGsjJbmu/dRCm32gcdZu1lqfDU0DLsnjk14GJpqpP5h6sEfSrdXyFcWzFzdjtLZLL6TfUWYNYX6CnjjRMv1zZ73877DPXt+vvi0Nvqld5CDTXM9ggDWwZKvluVGn7sTyZdwtWLvs1qK4nui7NLfENtBrUi/GOWsxoFa9tmfeeX/cticzzQcUdDNkfaDgmBa/C7lyjlkwyDGvhYdBHycSEJJ8rxncjBGKl79mpWlK0YTsppgD5eXWZSK7gC3PecRqQ7Jri3aBAWymlM7wfJYP8Z5ocEEq7+BLKSk0Z+Npj8PqJkQ0mw96QC5kNY2LfnGTPbBPpyBZRfZh3F2x1fLN+JQN+IgMpzLW2Oce5HWlPn9iTgCiZU/7mRDJKe8wI4gSg+Mf6hWqlC+NIcMOdAcrfvobzx3PVDHletTd0xgdgGEJpOvtpkYCqcdDbFX4kyEntMaMmp32XBcgsUy1b09uHMB2klzncwXAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQereGW4ZbKx3BV6f/PqkgSYAwcqDeq645K3JJOx9su/j9qDcAGxJN1CqIJjkYFBI+/2euykTaCvIqUMhljoDjeJeE]
+  cdn_password: ENC[PKCS7,MIIDCAYJKoZIhvcNAQcDoIIC+TCCAvUCAQAxggKQMIICjAIBADB0MFwxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxLTArBgNVBAMMJGludGVybmFsLXN0bzMtdGVzdC1kYi0xLmNkbi5zdW5ldC5zZQIUbbUXduFvDLw3OUVWiGrIvFBkkJMwDQYJKoZIhvcNAQEBBQAEggIAC2YvfQOuluAonjkj0iM5DABoSNLXLyjlXfkltBOtAWzlFAQfQNlKd9cArL0qthcge+4AYun9edbyrmKjBAqVYIjPZXMaUjN9HXa07vBwUaHUXUP/rSxL6JYWKAvZTUCnSS+rb/nUM8BAAodk2xNnDrd0H/VN2oBQMkFvWJbCX2/NS9zejr4BpcGTTLjr1GXOuRMwORXwNTHVYZBbZzltnXMRClcdUe9oeIfC2W0BJDTlvAsqVN4DAz985hP8b5vch3uTd59Qzr7pIqpdno8hoI75zdVZ+xeH31rYw5/wqHmsvQK3gvVTtp5zmO4lSWwhiyGfICsX1w/8Fa8j/qR17dfqzkWVWJVN4DN2O/iT1muMtP4WP4j5xvWusE1viW4qGnoFlheo9YVTCP08FLMn0BMyO/7r1AZNG0oDNhTw/DcGYRc0q1bF8L6LjprJrl8Ou7fZbxWcIuQ8UMC+aJPOI0vingTip7/nKhGIBSiplxxKPH3jB2G7NXnimi4sxgAXsXwSSHGTMZH6q46Kc7YrtzT6Vur0W8onQgqFw6Hg5kgybyrdU68skxqAHIQEV3bZ68e5f3MyKY5HxSse8IIngAQdF6mOvOf6JB3zQ98m/JXDDV9FvaMLXSu4iUMGmoHJDO6xmMLaPUamJobM7SFA+0gPa3hAV7fejDdzhh6bLFYwXAYJKoZIhvcNAQcBMB0GCWCGSAFlAwQBKgQQCT1soHO5e0vaqTWVkkyhS4Awdqh/mowa1LW46di/aaHZp0wPie3eEZnaDsKrIAo4yMFi/Sd0QCXF8YjicLFt0vK5]