From 790ed6d39a2011536c5b0811bbb3f336b5f30180 Mon Sep 17 00:00:00 2001
From: Maria Haider <mariah@sunet.se>
Date: Tue, 12 Dec 2023 09:58:04 +0100
Subject: [PATCH] Changing MDQ in natmd-1.komreg.net
Ref: SC-1384
---
global/overlay/etc/puppet/cosmos-rules.yaml | 21 +++++---
.../opt/mdqp/post.d/40-create-transitive | 53 +++++++++++++++++++
.../opt/mdqp/post.d/90-cleanup-stale-files | 6 +++
.../overlay/opt/mdqp/post.d/91-template.sh | 5 ++
natmd-common/overlay/opt/mdqp/post.d/97-rsync | 23 ++++++++
.../opt/mdqp/post.d/99-check-consistency | 35 ++++++++++++
.../overlay/opt/mdqp/pre.d/00-daily-clean | 14 +++++
.../opt/mdqp/pre.d/10-fetching-metadata | 18 +++++++
.../overlay/opt/mdqp/pre.d/20-restart-pyff | 34 ++++++++++++
natmd-common/overlay/opt/mdqp/rsync_lock | 1 +
natmd-common/overlay/root/.ssh/config | 6 +++
11 files changed, 210 insertions(+), 6 deletions(-)
create mode 100755 natmd-common/overlay/opt/mdqp/post.d/40-create-transitive
create mode 100755 natmd-common/overlay/opt/mdqp/post.d/90-cleanup-stale-files
create mode 100755 natmd-common/overlay/opt/mdqp/post.d/91-template.sh
create mode 100755 natmd-common/overlay/opt/mdqp/post.d/97-rsync
create mode 100755 natmd-common/overlay/opt/mdqp/post.d/99-check-consistency
create mode 100755 natmd-common/overlay/opt/mdqp/pre.d/00-daily-clean
create mode 100755 natmd-common/overlay/opt/mdqp/pre.d/10-fetching-metadata
create mode 100755 natmd-common/overlay/opt/mdqp/pre.d/20-restart-pyff
create mode 100644 natmd-common/overlay/opt/mdqp/rsync_lock
create mode 100644 natmd-common/overlay/root/.ssh/config
diff --git a/global/overlay/etc/puppet/cosmos-rules.yaml b/global/overlay/etc/puppet/cosmos-rules.yaml
index 76069ae1..26c2aa1a 100644
--- a/global/overlay/etc/puppet/cosmos-rules.yaml
+++ b/global/overlay/etc/puppet/cosmos-rules.yaml
@@ -447,12 +447,21 @@ natmd-1.komreg.net:
version: '5:23.0.6-1~ubuntu.20.04~focal'
metadatamgrs:
konsulter:
- eidas_hsm_client:
- md_signer:
- name: natmd-prod
- dest_host: natpub-1.komreg.net
- version: 1.1.5-eidas
- md_repo_client:
+ sunet::metadata::pyff_compose:
+ pyff_imagetag: swamid-2023-10-24
+ pyff_pipeline: natmd-prod-ng.fd
+ pyff_update_frequency: 1800
+ pyff_extra_volumes:
+ - "/etc/credentials:/etc/credentials"
+ hsm_client: true
+ luna_imagetag: 7.4-dev
+ sunet::metadata::mdqp:
+ imagetag: v2023-11-01-01
+ mdq_service: http://pyff_pyff_1:8080
+ sunet::metadata::metadata_repo:
+ hostname: r1.komreg.net
+ repo: komreg-metadata.git
+ signed_repo: true
natmd-test-1.komreg.net:
sunetops:
diff --git a/natmd-common/overlay/opt/mdqp/post.d/40-create-transitive b/natmd-common/overlay/opt/mdqp/post.d/40-create-transitive
new file mode 100755
index 00000000..2c36ee07
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/post.d/40-create-transitive
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -e
+
+base_url="http://localhost:8080"
+base_dir="/opt/mdqp/work/signed_metadata"
+
+tmpfile=$(mktemp /tmp/"$(basename "$0")".XXXXXX)
+
+
+for expression in "role-sp" "role-idp" "index.html"; do
+ echo "Fetching up ${expression}"
+
+ case $expression in
+ role-sp)
+ url="role/sp.xml"
+ file="/role/sp.xml"
+ ;;
+ role-idp)
+ url="role/idp.xml"
+ file="/role/idp.xml"
+ ;;
+ index.html)
+ url="entities/"
+ file="/entities/index.html"
+ ;;
+
+ *)
+ echo "Unsupported expression ${expression}"
+ exit 1
+ ;;
+ esac
+
+ curl ${base_url}/${url} -o "${tmpfile}"
+
+ case $file in
+ *.json)
+ jq -e .[].entityID "${tmpfile}" 1> /dev/null
+ ;;
+ *.xml|*.html)
+ xmlstarlet sel -t -v "//md:EntityDescriptor/@entityID" -n -m "//*[local-name()='EntityDescriptor']" -v "@entityID" -n "${tmpfile}" 1>/dev/null
+ ;;
+ *)
+ echo "Unsupported file type (${file}))"
+ exit 1
+ ;;
+ esac
+
+ full_path=${base_dir}${file}
+ mkdir -p "$(dirname ${full_path})"
+ mv "${tmpfile}" "${full_path}"
+
+done
diff --git a/natmd-common/overlay/opt/mdqp/post.d/90-cleanup-stale-files b/natmd-common/overlay/opt/mdqp/post.d/90-cleanup-stale-files
new file mode 100755
index 00000000..df757f60
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/post.d/90-cleanup-stale-files
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Cleanup stale files (removed files that the main script missout of removing (race))
+find /opt/mdqp/work/seen_metadata/ -type f -mtime +2 -print -delete
diff --git a/natmd-common/overlay/opt/mdqp/post.d/91-template.sh b/natmd-common/overlay/opt/mdqp/post.d/91-template.sh
new file mode 100755
index 00000000..9d248184
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/post.d/91-template.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rsync -a --chmod=D0755,F0644 /etc/mirror-mdq/template/ /opt/mdqp/work/signed_metadata/
+
+attr -q -s Content-Type -V "application/xml" /opt/mdqp/work/signed_metadata/entities/index.html
diff --git a/natmd-common/overlay/opt/mdqp/post.d/97-rsync b/natmd-common/overlay/opt/mdqp/post.d/97-rsync
new file mode 100755
index 00000000..7b5c00de
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/post.d/97-rsync
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+lock_file="/opt/mdqp/rsync_lock"
+
+if [ ! -r "${lock_file}" ]; then
+ echo "No lock_file (${lock_file}) found. Skipping rsync to publishers."
+ exit 0
+else
+ sync_host=$(cat "${lock_file}")
+fi
+
+my_hostname=$(hostname -f)
+if [ "${sync_host}x" != "${my_hostname}x" ]; then
+ echo "I'm not the sync host (${sync_host}). Skipping rsync to publishers."
+ exit 0
+fi
+
+for publisher in natpub-1.komreg.net; do
+ echo "rsync to ${publisher}"
+ rsync -a --exclude "/status/" --delete /opt/mdqp/work/signed_metadata/ ${publisher}:
+done
diff --git a/natmd-common/overlay/opt/mdqp/post.d/99-check-consistency b/natmd-common/overlay/opt/mdqp/post.d/99-check-consistency
new file mode 100755
index 00000000..8939b9d4
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/post.d/99-check-consistency
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+
+metadata_dir="/opt/mdqp/work/incoming_metadata/"
+signed_dir="/opt/mdqp/work/signed_metadata/entities/"
+
+loaded_entites_in_pyff=$(curl -s localhost:8080/api/status | jq .store.size)
+entities_on_disk=$(find ${metadata_dir} -type f -printf "%f\n" |sort |uniq | wc -l)
+
+exit_status=0
+if [ "${loaded_entites_in_pyff}" -ne "${entities_on_disk}" ]; then
+ echo "Pyff has ${loaded_entites_in_pyff} entites loaded but there are ${entities_on_disk} entities in ${metadata_dir}. Please investigate pyffs logs."
+ exit_status=1
+fi
+
+incoming_files=$(find ${metadata_dir}/* -type f | wc -l)
+signed_files=$(find ${signed_dir} -type f ! -name 'index.html' | wc -l)
+
+if [ "${incoming_files}" -ne "${signed_files}" ]; then
+ exit_status=1
+ echo "The incoming metadata dir (${metadata_dir}) contains ${incoming_files} and the signed metadata dir (${signed_dir}) contains ${signed_files}. That ain't right. Please investigate."
+ echo "The following files might help you investigate:"
+ entities_file=$(mktemp)
+
+ cd /opt/mdqp/work
+ grep entityID signed_metadata/entities/%7Bsha1%7D* | sed -e 's/.*entityID="\(.*\)" ID=.*/\1.xml/' -e 's/".*.xml/.xml/' -e 's@https://@@' -e 's@http://@@' | tr ':/?=' '----' | sort > "${entities_file}"
+ find incoming_metadata/ -type f -printf '%f\n' | sort | comm -3 - "${entities_file}"
+ if [ -f "${entities_file}" ]; then
+ rm "${entities_file}"
+ fi
+
+fi
+exit ${exit_status}
diff --git a/natmd-common/overlay/opt/mdqp/pre.d/00-daily-clean b/natmd-common/overlay/opt/mdqp/pre.d/00-daily-clean
new file mode 100755
index 00000000..ff5402bb
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/pre.d/00-daily-clean
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+sync_file="/opt/mdqp/work/full_sync"
+if [ -f ${sync_file} ]; then
+
+ if [ "$(date -r ${sync_file} +%Y-%m-%d)" != "$(date +%Y-%m-%d)" ]; then
+ rm "${sync_file}"
+ fi
+fi
+
+
+
diff --git a/natmd-common/overlay/opt/mdqp/pre.d/10-fetching-metadata b/natmd-common/overlay/opt/mdqp/pre.d/10-fetching-metadata
new file mode 100755
index 00000000..2e6267f8
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/pre.d/10-fetching-metadata
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Verify and update the repo. Previously run by cron but to mitigate/remove
+# race condion it moved here.
+cd /var/cache/metadata_r1.komreg.net && make update
+
+cd /opt/metadata
+
+incoming_dir=/opt/mdqp/work/incoming_metadata
+mkdir -p ${incoming_dir}
+tmp_dir=$(mktemp -d)
+
+rsync -a qa-idp/*.xml qa-sp/*.xml ${tmp_dir}/
+rsync -a --delete ${tmp_dir}/ ${incoming_dir}/
+
+rm -rf "${tmp_dir}"
diff --git a/natmd-common/overlay/opt/mdqp/pre.d/20-restart-pyff b/natmd-common/overlay/opt/mdqp/pre.d/20-restart-pyff
new file mode 100755
index 00000000..22acc7ef
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/pre.d/20-restart-pyff
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+
+metadata_dir="/opt/mdqp/work/incoming_metadata/"
+entities_on_disk=$(find ${metadata_dir} -type f -printf "%f\n" |sort |uniq | wc -l)
+
+systemctl restart sunet-pyff.service
+
+# Pyff is never faster then 20 sec. Prevent exceptions in log by not trying
+# earlier.
+sleep 25
+
+# Give pyff some time to start
+happy_pyff=0
+for try in {1..30}; do
+ entitites=$(curl -s 'localhost:8080/api/status' | jq .store.size 2> /dev/null || echo 0)
+ if [ -n "${entitites}" ] && [ "${entitites}" != "0" ]; then
+ loaded_entites_in_percent=$(python3 -c "print(int(${entitites} / ${entities_on_disk} * 100))")
+ if [ "${loaded_entites_in_percent}" -ge "96" ]; then
+ happy_pyff=200
+ break
+ fi
+ fi
+ sleep "${try}";
+done
+
+if [ "${happy_pyff}" != "200" ]; then
+ echo "couldn't restart pyff - exit"
+ exit 1
+fi
+
+# Hängslen och livrem
+sleep 10
diff --git a/natmd-common/overlay/opt/mdqp/rsync_lock b/natmd-common/overlay/opt/mdqp/rsync_lock
new file mode 100644
index 00000000..543e80f0
--- /dev/null
+++ b/natmd-common/overlay/opt/mdqp/rsync_lock
@@ -0,0 +1 @@
+md1.komreg.net
diff --git a/natmd-common/overlay/root/.ssh/config b/natmd-common/overlay/root/.ssh/config
new file mode 100644
index 00000000..e07f051f
--- /dev/null
+++ b/natmd-common/overlay/root/.ssh/config
@@ -0,0 +1,6 @@
+Host natpub-1.komreg.net
+IdentityFile /root/.ssh/default
+Host natpub-2.komreg.net
+IdentityFile /root/.ssh/default
+Host r1.komreg.net
+IdentityFile /root/.ssh/komreg