diff --git a/global/overlay/etc/puppet/cosmos-rules.yaml b/global/overlay/etc/puppet/cosmos-rules.yaml index 1a885cbd..4aaf999a 100644 --- a/global/overlay/etc/puppet/cosmos-rules.yaml +++ b/global/overlay/etc/puppet/cosmos-rules.yaml @@ -747,18 +747,26 @@ p2.qa.komreg.net: md1.komreg.net: sunetops: - sunet_iaas_cloud: autoupdate: + eidas_metadata_key: eid::dockerhost: - version: '5:23.0.6-1~ubuntu.20.04~focal' + version: '5:24.0.7-1~ubuntu.20.04~focal' + docker_repo: stable metadatamgrs: konsulter: - eidas_metadata_key: - md_signer: - name: natmd-qa - dest_host: p1.komreg.net - version: 1.1.5-eidas - md_repo_client: + sunet::metadata::pyff_compose: + pyff_imagetag: swamid-2023-10-24 + pyff_pipeline: natmd-qa-ng.fd + pyff_update_frequency: 1800 + pyff_extra_volumes: + - "/etc/credentials:/etc/credentials" + sunet::metadata::mdqp: + imagetag: v2023-09-13-01 + mdq_service: http://pyff_pyff_1:8080 + sunet::metadata::metadata_repo: + hostname: r1.komreg.net + repo: komreg-metadata.git + signed_repo: true md-eu1.qa.komreg.net: sunetops: diff --git a/md1.komreg.net/overlay/opt/mdqp/post.d/40-create-transitive b/md1.komreg.net/overlay/opt/mdqp/post.d/40-create-transitive new file mode 100755 index 00000000..2c36ee07 --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/post.d/40-create-transitive @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -e + +base_url="http://localhost:8080" +base_dir="/opt/mdqp/work/signed_metadata" + +tmpfile=$(mktemp /tmp/"$(basename "$0")".XXXXXX) + + +for expression in "role-sp" "role-idp" "index.html"; do + echo "Fetching up ${expression}" + + case $expression in + role-sp) + url="role/sp.xml" + file="/role/sp.xml" + ;; + role-idp) + url="role/idp.xml" + file="/role/idp.xml" + ;; + index.html) + url="entities/" + file="/entities/index.html" + ;; + + *) + echo "Unsupported expression ${expression}" + exit 1 + ;; + esac + + curl ${base_url}/${url} -o "${tmpfile}" + + case $file in + *.json) + jq -e .[].entityID "${tmpfile}" 1> /dev/null + ;; + *.xml|*.html) + xmlstarlet sel -t -v "//md:EntityDescriptor/@entityID" -n -m "//*[local-name()='EntityDescriptor']" -v "@entityID" -n "${tmpfile}" 1>/dev/null + ;; + *) + echo "Unsupported file type (${file}))" + exit 1 + ;; + esac + + full_path=${base_dir}${file} + mkdir -p "$(dirname ${full_path})" + mv "${tmpfile}" "${full_path}" + +done diff --git a/md1.komreg.net/overlay/opt/mdqp/post.d/90-cleanup-stale-files b/md1.komreg.net/overlay/opt/mdqp/post.d/90-cleanup-stale-files new file mode 100755 index 00000000..df757f60 --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/post.d/90-cleanup-stale-files @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +# Cleanup stale files (removed files that the main script missout of removing (race)) +find /opt/mdqp/work/seen_metadata/ -type f -mtime +2 -print -delete diff --git a/md1.komreg.net/overlay/opt/mdqp/post.d/91-template.sh b/md1.komreg.net/overlay/opt/mdqp/post.d/91-template.sh new file mode 100755 index 00000000..9d248184 --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/post.d/91-template.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rsync -a --chmod=D0755,F0644 /etc/mirror-mdq/template/ /opt/mdqp/work/signed_metadata/ + +attr -q -s Content-Type -V "application/xml" /opt/mdqp/work/signed_metadata/entities/index.html diff --git a/md1.komreg.net/overlay/opt/mdqp/post.d/97-rsync b/md1.komreg.net/overlay/opt/mdqp/post.d/97-rsync new file mode 100755 index 00000000..0d02f6c6 --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/post.d/97-rsync @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +lock_file="/opt/mdqp/rsync_lock" + +if [ ! -r "${lock_file}" ]; then + echo "No lock_file (${lock_file}) found. Skipping rsync to publishers." + exit 0 +else + sync_host=$(cat "${lock_file}") +fi + +my_hostname=$(hostname -f) +if [ "${sync_host}x" != "${my_hostname}x" ]; then + echo "I'm not the sync host (${sync_host}). Skipping rsync to publishers." + exit 0 +fi + +for publisher in p1.komreg.net; do + echo "rsync to ${publisher}" + rsync -a --exclude "/status/" --delete /opt/mdqp/work/signed_metadata/ ${publisher}: +done diff --git a/md1.komreg.net/overlay/opt/mdqp/post.d/99-check-consistency b/md1.komreg.net/overlay/opt/mdqp/post.d/99-check-consistency new file mode 100755 index 00000000..8939b9d4 --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/post.d/99-check-consistency @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e + + +metadata_dir="/opt/mdqp/work/incoming_metadata/" +signed_dir="/opt/mdqp/work/signed_metadata/entities/" + +loaded_entites_in_pyff=$(curl -s localhost:8080/api/status | jq .store.size) +entities_on_disk=$(find ${metadata_dir} -type f -printf "%f\n" |sort |uniq | wc -l) + +exit_status=0 +if [ "${loaded_entites_in_pyff}" -ne "${entities_on_disk}" ]; then + echo "Pyff has ${loaded_entites_in_pyff} entites loaded but there are ${entities_on_disk} entities in ${metadata_dir}. Please investigate pyffs logs." + exit_status=1 +fi + +incoming_files=$(find ${metadata_dir}/* -type f | wc -l) +signed_files=$(find ${signed_dir} -type f ! -name 'index.html' | wc -l) + +if [ "${incoming_files}" -ne "${signed_files}" ]; then + exit_status=1 + echo "The incoming metadata dir (${metadata_dir}) contains ${incoming_files} and the signed metadata dir (${signed_dir}) contains ${signed_files}. That ain't right. Please investigate." + echo "The following files might help you investigate:" + entities_file=$(mktemp) + + cd /opt/mdqp/work + grep entityID signed_metadata/entities/%7Bsha1%7D* | sed -e 's/.*entityID="\(.*\)" ID=.*/\1.xml/' -e 's/".*.xml/.xml/' -e 's@https://@@' -e 's@http://@@' | tr ':/?=' '----' | sort > "${entities_file}" + find incoming_metadata/ -type f -printf '%f\n' | sort | comm -3 - "${entities_file}" + if [ -f "${entities_file}" ]; then + rm "${entities_file}" + fi + +fi +exit ${exit_status} diff --git a/md1.komreg.net/overlay/opt/mdqp/pre.d/00-daily-clean b/md1.komreg.net/overlay/opt/mdqp/pre.d/00-daily-clean new file mode 100755 index 00000000..ff5402bb --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/pre.d/00-daily-clean @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +sync_file="/opt/mdqp/work/full_sync" +if [ -f ${sync_file} ]; then + + if [ "$(date -r ${sync_file} +%Y-%m-%d)" != "$(date +%Y-%m-%d)" ]; then + rm "${sync_file}" + fi +fi + + + diff --git a/md1.komreg.net/overlay/opt/mdqp/pre.d/10-fetching-metadata b/md1.komreg.net/overlay/opt/mdqp/pre.d/10-fetching-metadata new file mode 100755 index 00000000..2e6267f8 --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/pre.d/10-fetching-metadata @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e + +# Verify and update the repo. Previously run by cron but to mitigate/remove +# race condion it moved here. +cd /var/cache/metadata_r1.komreg.net && make update + +cd /opt/metadata + +incoming_dir=/opt/mdqp/work/incoming_metadata +mkdir -p ${incoming_dir} +tmp_dir=$(mktemp -d) + +rsync -a qa-idp/*.xml qa-sp/*.xml ${tmp_dir}/ +rsync -a --delete ${tmp_dir}/ ${incoming_dir}/ + +rm -rf "${tmp_dir}" diff --git a/md1.komreg.net/overlay/opt/mdqp/pre.d/20-restart-pyff b/md1.komreg.net/overlay/opt/mdqp/pre.d/20-restart-pyff new file mode 100755 index 00000000..22acc7ef --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/pre.d/20-restart-pyff @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -e + +metadata_dir="/opt/mdqp/work/incoming_metadata/" +entities_on_disk=$(find ${metadata_dir} -type f -printf "%f\n" |sort |uniq | wc -l) + +systemctl restart sunet-pyff.service + +# Pyff is never faster then 20 sec. Prevent exceptions in log by not trying +# earlier. +sleep 25 + +# Give pyff some time to start +happy_pyff=0 +for try in {1..30}; do + entitites=$(curl -s 'localhost:8080/api/status' | jq .store.size 2> /dev/null || echo 0) + if [ -n "${entitites}" ] && [ "${entitites}" != "0" ]; then + loaded_entites_in_percent=$(python3 -c "print(int(${entitites} / ${entities_on_disk} * 100))") + if [ "${loaded_entites_in_percent}" -ge "96" ]; then + happy_pyff=200 + break + fi + fi + sleep "${try}"; +done + +if [ "${happy_pyff}" != "200" ]; then + echo "couldn't restart pyff - exit" + exit 1 +fi + +# Hängslen och livrem +sleep 10 diff --git a/md1.komreg.net/overlay/opt/mdqp/rsync_lock b/md1.komreg.net/overlay/opt/mdqp/rsync_lock new file mode 100644 index 00000000..543e80f0 --- /dev/null +++ b/md1.komreg.net/overlay/opt/mdqp/rsync_lock @@ -0,0 +1 @@ +md1.komreg.net diff --git a/md1.komreg.net/overlay/root/.ssh/config b/md1.komreg.net/overlay/root/.ssh/config new file mode 100644 index 00000000..c4d1682c --- /dev/null +++ b/md1.komreg.net/overlay/root/.ssh/config @@ -0,0 +1,4 @@ +Host p1.komreg.net +IdentityFile /root/.ssh/default +Host r1.komreg.net +IdentityFile /root/.ssh/komreg