New MDQ for SC QA

Ref: SC-1384

global/overlay/etc/puppet/cosmos-rules.yaml

@@ -747,18 +747,26 @@ p2.qa.komreg.net:
 
 md1.komreg.net:
    sunetops:
-   sunet_iaas_cloud:
    autoupdate:
+   eidas_metadata_key:
    eid::dockerhost:
-      version: '5:23.0.6-1~ubuntu.20.04~focal'
+      version: '5:24.0.7-1~ubuntu.20.04~focal'
+      docker_repo: stable
    metadatamgrs:
    konsulter:
-   eidas_metadata_key:
+   sunet::metadata::pyff_compose:
-   md_signer:
+      pyff_imagetag: swamid-2023-10-24
-      name: natmd-qa
+      pyff_pipeline: natmd-qa-ng.fd
-      dest_host: p1.komreg.net
+      pyff_update_frequency: 1800
-      version: 1.1.5-eidas
+      pyff_extra_volumes:
-   md_repo_client:
+        - "/etc/credentials:/etc/credentials"
+   sunet::metadata::mdqp:
+      imagetag: v2023-09-13-01
+      mdq_service: http://pyff_pyff_1:8080
+   sunet::metadata::metadata_repo:
+      hostname: r1.komreg.net
+      repo: komreg-metadata.git
+      signed_repo: true
 
 md-eu1.qa.komreg.net:
    sunetops:

md1.komreg.net/overlay/opt/mdqp/post.d/40-create-transitive

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -e
+
+base_url="http://localhost:8080"
+base_dir="/opt/mdqp/work/signed_metadata"
+
+tmpfile=$(mktemp /tmp/"$(basename "$0")".XXXXXX)
+
+
+for expression in "role-sp" "role-idp" "index.html"; do
+    echo "Fetching up ${expression}"
+
+    case $expression in
+      role-sp)
+        url="role/sp.xml"
+        file="/role/sp.xml"
+        ;;
+      role-idp)
+        url="role/idp.xml"
+        file="/role/idp.xml"
+        ;;
+      index.html)
+        url="entities/"
+        file="/entities/index.html"
+        ;;
+
+      *)
+        echo "Unsupported expression ${expression}"
+        exit 1
+        ;;
+    esac
+
+    curl ${base_url}/${url} -o "${tmpfile}"
+
+    case $file in
+      *.json)
+        jq -e .[].entityID "${tmpfile}" 1> /dev/null
+        ;;
+      *.xml|*.html)
+        xmlstarlet sel -t -v "//md:EntityDescriptor/@entityID" -n -m "//*[local-name()='EntityDescriptor']" -v "@entityID" -n "${tmpfile}" 1>/dev/null
+        ;;
+      *)
+        echo "Unsupported file type (${file}))"
+        exit 1
+        ;;
+     esac
+
+    full_path=${base_dir}${file}
+    mkdir -p "$(dirname ${full_path})"
+    mv "${tmpfile}" "${full_path}"
+
+done

md1.komreg.net/overlay/opt/mdqp/post.d/90-cleanup-stale-files

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Cleanup stale files (removed files that the main script missout of removing (race))
+find /opt/mdqp/work/seen_metadata/ -type f -mtime +2 -print -delete

md1.komreg.net/overlay/opt/mdqp/post.d/91-template.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rsync -a --chmod=D0755,F0644 /etc/mirror-mdq/template/ /opt/mdqp/work/signed_metadata/
+
+attr -q -s Content-Type -V "application/xml" /opt/mdqp/work/signed_metadata/entities/index.html

md1.komreg.net/overlay/opt/mdqp/post.d/97-rsync

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+lock_file="/opt/mdqp/rsync_lock"
+
+if [ ! -r "${lock_file}" ]; then
+    echo "No lock_file (${lock_file}) found. Skipping rsync to publishers."
+    exit 0
+else
+    sync_host=$(cat "${lock_file}")
+fi
+
+my_hostname=$(hostname -f)
+if [ "${sync_host}x" != "${my_hostname}x" ]; then
+    echo "I'm not the sync host (${sync_host}). Skipping rsync to publishers."
+    exit 0
+fi
+
+for publisher in p1.komreg.net; do
+    echo "rsync to ${publisher}"
+    rsync -a --exclude "/status/" --delete /opt/mdqp/work/signed_metadata/ ${publisher}:
+done

md1.komreg.net/overlay/opt/mdqp/post.d/99-check-consistency

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+
+metadata_dir="/opt/mdqp/work/incoming_metadata/"
+signed_dir="/opt/mdqp/work/signed_metadata/entities/"
+
+loaded_entites_in_pyff=$(curl -s localhost:8080/api/status | jq  .store.size)
+entities_on_disk=$(find ${metadata_dir} -type f -printf "%f\n" |sort |uniq | wc -l)
+
+exit_status=0
+if [ "${loaded_entites_in_pyff}" -ne "${entities_on_disk}" ]; then
+   echo "Pyff has ${loaded_entites_in_pyff} entites loaded but there are ${entities_on_disk} entities in ${metadata_dir}. Please investigate pyffs logs."
+   exit_status=1
+fi
+
+incoming_files=$(find ${metadata_dir}/* -type f  | wc -l)
+signed_files=$(find ${signed_dir} -type f ! -name 'index.html' | wc -l)
+
+if [ "${incoming_files}" -ne "${signed_files}" ]; then
+   exit_status=1
+   echo "The incoming metadata dir (${metadata_dir}) contains ${incoming_files} and the signed metadata dir (${signed_dir}) contains ${signed_files}. That ain't right. Please investigate."
+   echo "The following files might help you investigate:"
+   entities_file=$(mktemp)
+
+   cd /opt/mdqp/work
+   grep entityID signed_metadata/entities/%7Bsha1%7D* | sed -e 's/.*entityID="\(.*\)" ID=.*/\1.xml/' -e 's/".*.xml/.xml/' -e 's@https://@@' -e 's@http://@@' | tr ':/?=' '----' | sort > "${entities_file}"
+   find incoming_metadata/ -type f -printf '%f\n' | sort | comm -3 - "${entities_file}"
+   if [ -f "${entities_file}" ]; then
+       rm "${entities_file}"
+   fi
+
+fi
+exit ${exit_status}

md1.komreg.net/overlay/opt/mdqp/pre.d/00-daily-clean

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+sync_file="/opt/mdqp/work/full_sync"
+if [ -f ${sync_file} ]; then
+
+    if [ "$(date -r ${sync_file} +%Y-%m-%d)" != "$(date +%Y-%m-%d)" ]; then
+        rm "${sync_file}"
+    fi
+fi
+
+
+

md1.komreg.net/overlay/opt/mdqp/pre.d/10-fetching-metadata

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Verify and update the repo. Previously run by cron but to mitigate/remove
+# race condion it moved here.
+cd /var/cache/metadata_r1.komreg.net && make update
+
+cd /opt/metadata
+
+incoming_dir=/opt/mdqp/work/incoming_metadata
+mkdir -p ${incoming_dir}
+tmp_dir=$(mktemp -d)
+
+rsync -a qa-idp/*.xml qa-sp/*.xml ${tmp_dir}/
+rsync -a --delete ${tmp_dir}/ ${incoming_dir}/
+
+rm -rf "${tmp_dir}"

md1.komreg.net/overlay/opt/mdqp/pre.d/20-restart-pyff

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+
+metadata_dir="/opt/mdqp/work/incoming_metadata/"
+entities_on_disk=$(find ${metadata_dir} -type f -printf "%f\n" |sort |uniq | wc -l)
+
+systemctl restart sunet-pyff.service
+
+# Pyff is never faster then 20 sec. Prevent exceptions in log by not trying
+# earlier.
+sleep 25
+
+# Give pyff some time to start
+happy_pyff=0
+for try in {1..30}; do
+    entitites=$(curl -s 'localhost:8080/api/status' | jq .store.size 2> /dev/null || echo 0)
+    if [ -n "${entitites}" ] && [ "${entitites}" != "0" ]; then
+        loaded_entites_in_percent=$(python3 -c "print(int(${entitites} / ${entities_on_disk} * 100))")
+        if [ "${loaded_entites_in_percent}" -ge "96" ]; then
+            happy_pyff=200
+            break
+        fi
+    fi
+    sleep "${try}";
+done
+
+if [ "${happy_pyff}" != "200" ]; then
+    echo "couldn't restart pyff - exit"
+    exit 1
+fi
+
+# Hängslen och livrem
+sleep 10

md1.komreg.net/overlay/opt/mdqp/rsync_lock

@@ -0,0 +1 @@
+md1.komreg.net

md1.komreg.net/overlay/root/.ssh/config

@@ -0,0 +1,4 @@
+Host p1.komreg.net
+IdentityFile /root/.ssh/default
+Host r1.komreg.net
+IdentityFile /root/.ssh/komreg