From 76ff59431ed0acdd4750ef44a85dd3b263f51431 Mon Sep 17 00:00:00 2001
From: Maria Haider <mariah@nordu.net>
Date: Wed, 17 Apr 2019 10:39:12 +0200
Subject: [PATCH] Scripts for creating hiera-gpg key and secrets

Compatible for ubuntu 18 hosts
---
 edit-secrets                    | 323 +++++++++++++++++++++-----------
 global/pre-tasks.d/040hiera-gpg |  38 ++--
 2 files changed, 240 insertions(+), 121 deletions(-)

diff --git a/edit-secrets b/edit-secrets
index 145255ba..0921be6d 100755
--- a/edit-secrets
+++ b/edit-secrets
@@ -1,89 +1,27 @@
 #!/bin/bash
+#
+# Script to edit secrets for a host.
+#
+# This script is used by an administrator on his/hers local machine. The
+# general principle is for this script to ssh to the target host, decrypt
+# the secrets and allow changes to be made, and then fetch the encrypted
+# secrets from the host and add it to the Cosmos repository on the
+# administrators machine.
+#
+# Funnily enough, this script will execute itself (with the argument
+# '--on-host') on the target host in order to do the decryption etc. Don't
+# allow this to confuse you and everything will be fine.
+#
 
 set -e
 umask 077
 
 LAST_OUTPUT_FILENAME="/root/.last_edit-secrets_output"
 
-if [ "x$1" = "x" ]; then
-    echo "Syntax: $0 -l OR fqdn"
-    exit 1
-fi
+test -d /dev/shm && export TMPDIR='/dev/shm'
 
-if [ "x$1" != "x-l" ]; then
-    host=$(echo $1 | sed -e 's!/*$!!')    # remove trailing slashes
-
-    if [ ! -d $host ]; then
-	echo "$0: No host-directory for '$host' found - execute in top-level cosmos dir"
-	exit 1
-    fi
-
-    # Execute this very script, on a remote host
-    TMPFILE=$(mktemp edit-secrets.$$.XXXXXXX)
-    if [ ! -f $TMPFILE ]; then
-	echo "$0: Failed creating temporary file"
-	exit 1
-    fi
-    TMPFILE2=$(mktemp edit-secrets.$$.XXXXXXX)
-    if [ ! -f $TMPFILE2 ]; then
-	echo "$0: Failed creating temporary file"
-	exit 1
-    fi
-
-    trap "rm -f $TMPFILE $TMPFILE2" EXIT
-
-    ssh -t root@$host /var/cache/cosmos/repo/edit-secrets -l
-    scp -q root@$host:$LAST_OUTPUT_FILENAME $TMPFILE
-
-    if grep ^"STATUS=UPDATED" $TMPFILE > /dev/null; then
-	# extract the path of the file that should be updated in the Cosmos repo
-        save_to="${host}/overlay/etc/hiera/data/secrets.yaml.asc"
-        mkdir -p "`dirname $save_to`"
-	# extract the GPG output
-	perl -e '$a = 0; while (<>) { $a = 1 if ($_ =~ /-+BEGIN PGP MESSAGE-+/);
-                 print $_ if $a; $a = 0 if ($_ =~ /-+END PGP MESSAGE-+/); }' < $TMPFILE > $TMPFILE2
-
-	if ! grep "END PGP MESSAGE" $TMPFILE2 > /dev/null; then
-	    echo "$0: Failed extracting PGP output from file $TMPFILE into $TMPFILE2"
-	    exit 1
-	fi
-	# use cat to preserve permissions etc.
-	cat $TMPFILE > $save_to
-	git add $save_to
-
-	echo ""
-	echo "$save_to updated"
-	echo ""
-    else
-	echo ""
-	echo "Not updated"
-	echo ""
-    fi
-
-    rm $TMPFILE $TMPFILE2
-
-    exit 0
-fi
-
-#
-# Local execution on a host
-#
-
-SECRETFILE=/etc/hiera/data/secrets.yaml.asc
-GNUPGHOME=/etc/hiera/gpg/
-export GNUPGHOME
-
-GPG=`which gpg2 || true`
-if [ ! -x "$GPG" ]; then
-    GPG=`which gpg || true`
-    if [ ! -x "$GPG" ]; then
-	echo "$0: gpg2 or gpg not found"
-	exit 1
-    fi
-fi
-
-TMPFILE=$(mktemp --tmpdir=/dev/shm)
-TMPFILE2=$(mktemp --tmpdir=/dev/shm)
+TMPFILE=$(mktemp edit-secrets.XXXXXXXXXX)
+TMPFILE2=$(mktemp edit-secrets.XXXXXXXXXX)
 
 if [ ! -f $TMPFILE ]; then
     echo "$TMPFILE"
@@ -98,47 +36,212 @@ fi
 
 trap "rm -f $TMPFILE $TMPFILE2" EXIT
 
-if ! $GPG --list-secret-keys | grep -q ^"sec\s"; then
-    echo "$0: Secret key does not exist (in $GNUPGHOME)."
-    echo ""
-    echo "Generate it with /var/cache/cosmos/model/pre-tasks.d/040hiera-gpg"
-    echo ""
+
+if [[ ! $1 ]]; then
+    # deliberately don't mention the --on-host argument
+    echo "Syntax: $0 fqdn"
     exit 1
 fi
 
-if [ -s $SECRETFILE ]; then
-    $GPG -d $SECRETFILE > $TMPFILE
-fi
 
-cp $TMPFILE $TMPFILE2
-sensible-editor $TMPFILE
-rm -f ${TMPFILE}~ ${TMPFILE2}~
+function edit_copy_and_commit()
+{
+    #
+    # This code runs on the administrators local machine
+    #
+    local host=$1
 
-echo ""
-echo ""
+    if [[ ${EDITOR} ]]; then
+	declare -r REMOTE_EDITOR="${EDITOR}"
+    else
+	declare -r REMOTE_EDITOR='/usr/bin/vim.tiny'
+    fi
 
-status=0
-cmp -s $TMPFILE $TMPFILE2 || status=1
-if [ $status -eq 0 ]; then
-    (
-	echo "STATUS=NOT_CHANGED"
-    )  > $LAST_OUTPUT_FILENAME
-    echo ""
-    echo "$0: No changes detected"
-else
-    # figure out this hosts gpg key id
-    recipient=$($GPG --list-secret-key | grep ^sec | head -1 | awk '{print $2}' | cut -d / -f 2)
+    # Execute this script, on a remote host
+    ssh -t root@"${host}" EDITOR="${REMOTE_EDITOR}" /var/cache/cosmos/repo/edit-secrets --on-host
+    scp -q root@"${host}:${LAST_OUTPUT_FILENAME}" ${TMPFILE}
 
-    save_to="`hostname --fqdn`/overlay${SECRETFILE}"
-    echo ""
-    (
-	echo "STATUS=UPDATED"
+    local save_to
+    if grep ^"STATUS=UPDATED" $TMPFILE > /dev/null; then
+        save_to="${host}/overlay/etc/hiera/data/secrets.yaml.asc"
+
+	# extract the GPG output
+	perl -e '$a = 0; while (<>) { $a = 1 if ($_ =~ /-+BEGIN PGP MESSAGE-+/);
+                 print $_ if $a; $a = 0 if ($_ =~ /-+END PGP MESSAGE-+/); }' < $TMPFILE > $TMPFILE2
+
+	if ! grep "END PGP MESSAGE" $TMPFILE2 > /dev/null; then
+	    echo "$0: Failed extracting PGP output from file $TMPFILE into $TMPFILE2"
+	    exit 1
+	fi
+    elif grep ^"STATUS=EYAML_UPDATED" $TMPFILE > /dev/null; then
+        save_to="${host}/overlay/etc/hiera/data/local.eyaml"
+
+	# remove the STATUS= line
+	grep -v '^STATUS=EYAML_UPDATED' $TMPFILE > $TMPFILE2
+
+	# check syntax
+	if [ -x $(dirname $0)/scripts/jsonyaml-no-output.py ]; then
+	    if ! $(dirname $0)/scripts/jsonyaml-no-output.py yaml $TMPFILE2; then
+		echo "$0: Error: $TMPFILE2 doesn't look like a YAML file"
+		exit 1
+	    fi
+	else
+	    echo "$0: Warning: Unable to check syntax of $TMPFILE2"
+	fi
+    else
 	echo ""
-    ) > $LAST_OUTPUT_FILENAME
-    $GPG --output - --armor --recipient $recipient --sign --encrypt $TMPFILE >> $LAST_OUTPUT_FILENAME
+	echo "Not updated"
+	echo ""
+
+	exit 0
+    fi
+
+    # use cat to preserve permissions etc.
+    mkdir -p "`dirname ${save_to}`"
+    cat $TMPFILE2 > "${save_to}"
+    git add "${save_to}"
+
+    if grep ^"STATUS=EYAML_UPDATED" $TMPFILE > /dev/null; then
+	git diff --cached "${save_to}"
+    fi
+
     echo ""
-    echo "GPG output saved in $LAST_OUTPUT_FILENAME - save it in Cosmos as"
+    echo "$save_to updated"
     echo ""
-    echo "  $save_to"
+
+    exit 0
+}
+
+function edit_file_on_host() {
+    #
+    # Local execution on a host
+    #
+
+    local SECRETFILE=/etc/hiera/data/secrets.yaml.asc
+    local EYAMLFILE=/etc/hiera/data/local.eyaml
+
+    if [ -f "${EYAMLFILE}" ]; then
+	edit_eyaml_file ${EYAMLFILE}
+    elif [ -f "${SECRETFILE}" ]; then
+	edit_gpg_file ${SECRETFILE}
+    elif [ -f /etc/hiera/eyaml/public_certkey.pkcs7.pem ]; then
+	# default to eyaml if the key exists and none of the secrets-file above exist
+	touch ${EYAMLFILE}
+	edit_eyaml_file ${EYAMLFILE}
+    fi
+}
+
+function edit_gpg_file()
+{
+    local SECRETFILE=$1
+
+    GNUPGHOME=/etc/hiera/gpg/
+    export GNUPGHOME
+
+    local GPG=`which gpg2 || true`
+    if [ ! -x "$GPG" ]; then
+	GPG=`which gpg || true`
+	if [ ! -x "$GPG" ]; then
+	    echo "$0: gpg2 or gpg not found"
+	    exit 1
+	fi
+    fi
+
+    if ! $GPG --list-secret-keys | grep -q ^"sec\s"; then
+	echo "$0: Secret key does not exist (in $GNUPGHOME)."
+	echo ""
+	echo "Generate it with /var/cache/cosmos/model/pre-tasks.d/040hiera-gpg"
+	echo ""
+	exit 1
+    fi
+
+    if [ -s $SECRETFILE ]; then
+	$GPG -d $SECRETFILE > $TMPFILE
+    fi
+
+    cp $TMPFILE $TMPFILE2
+    sensible-editor $TMPFILE
+    rm -f ${TMPFILE}~ ${TMPFILE2}~
+
     echo ""
+    echo ""
+
+    local status=0
+    cmp -s $TMPFILE $TMPFILE2 || status=1
+    if [ $status -eq 0 ]; then
+	(
+	    echo "STATUS=NOT_CHANGED"
+	)  > $LAST_OUTPUT_FILENAME
+	echo ""
+	echo "$0: No changes detected"
+    else
+	# figure out this hosts gpg key id
+	if lsb_release -r | grep -q 18.04; then
+	    recipient=$($GPG --list-secret-keys | grep -A1 '^sec' | tail -1 | awk '{print $1}')
+	else
+	    recipient=$($GPG --list-secret-key | grep ^sec | head -1 | awk '{print $2}' | cut -d / -f 2)
+	fi
+
+	save_to="`hostname --fqdn`/overlay${SECRETFILE}"
+	echo ""
+	(
+	    echo "STATUS=UPDATED"
+	    echo ""
+	) > $LAST_OUTPUT_FILENAME
+	$GPG --output - --armor --recipient $recipient --sign --encrypt $TMPFILE >> $LAST_OUTPUT_FILENAME
+	echo ""
+	echo "GPG output saved in $LAST_OUTPUT_FILENAME - save it in Cosmos as"
+	echo ""
+	echo "  $save_to"
+	echo ""
+    fi
+}
+
+function edit_eyaml_file()
+{
+    local EYAMLFILE=$1
+
+    local FQDN=$(hostname --fqdn)
+    local privkey='/etc/hiera/eyaml/private_key.pkcs7.pem'
+    local pubkey='/etc/hiera/eyaml/public_certkey.pkcs7.pem'
+    for f in $privkey $pubkey; do
+	test -f "${f}" || { echo "$0: eyaml key file ${f} not found"; exit 1; }
+    done
+
+    # save source file for comparision afterwards
+    cp "${EYAMLFILE}" "${TMPFILE}"
+    eyaml edit --pkcs7-private-key "${privkey}" --pkcs7-public-key "${pubkey}" "${EYAMLFILE}"
+
+    local status=0
+    cmp -s "${EYAMLFILE}" $TMPFILE || status=1
+    if [ $status -eq 0 ]; then
+	(
+	    echo "STATUS=NOT_CHANGED"
+	)  > $LAST_OUTPUT_FILENAME
+	echo ""
+	echo "$0: No changes detected"
+    else
+	echo ""
+	(
+	    echo "STATUS=EYAML_UPDATED"
+	    echo ""
+	) > $LAST_OUTPUT_FILENAME
+	cat "${EYAMLFILE}" >> $LAST_OUTPUT_FILENAME
+    fi
+}
+
+
+if [[ $1 == '--on-host' ]]; then
+    edit_file_on_host
+else
+    host=$(echo $1 | sed -e 's!/*$!!')    # remove trailing slashes
+
+    if [ ! -d $host ]; then
+	echo "$0: No host-directory for '$host' found - execute in top-level cosmos dir"
+	exit 1
+    fi
+
+    edit_copy_and_commit $host
 fi
+
+exit 0
diff --git a/global/pre-tasks.d/040hiera-gpg b/global/pre-tasks.d/040hiera-gpg
index aed6dbe9..6f6be9d8 100755
--- a/global/pre-tasks.d/040hiera-gpg
+++ b/global/pre-tasks.d/040hiera-gpg
@@ -6,25 +6,41 @@
 
 set -e
 
+EYAMLDIR=/etc/hiera/eyaml
 GNUPGHOME=/etc/hiera/gpg
 export GNUPGHOME
 
+
+# There is no hiera-eyaml on Ubuntu < 16.04
+if [ "x`lsb_release -r | awk '{print $NF}'`" != "x12.04" -a "x`lsb_release -r | awk '{print $NF}'`" != "x14.04"  ]; then
+    if [ ! -f /usr/bin/eyaml ]; then
+        apt-get update
+        apt-get -y install hiera-eyaml
+    fi
+fi
+
+if [ -f /usr/bin/eyaml ]; then
+    # Create eyaml keypair if eyaml is installed but there are no keys
+    if [ ! -f ${EYAMLDIR}/public_certkey.pkcs7.pem -o ! -f ${EYAMLDIR}/private_key.pkcs7.pem ]; then
+	# hiera-eyaml wants a certificate and public key, not just a public key oddly enough
+	echo "$0: Generating eyaml key in ${EYAMLDIR} - this might take a while..."
+	mkdir -p ${EYAMLDIR}
+	openssl req -x509 -newkey rsa:4096 -keyout ${EYAMLDIR}/private_key.pkcs7.pem \
+		-out ${EYAMLDIR}/public_certkey.pkcs7.pem -days 3653 -nodes -sha256 \
+		-subj "/C=SE/O=SUNET/OU=EYAML/CN=`hostname`"
+	rm -f ${EYAMLDIR}/public_key.pkcs7.pem  # cleanup
+    fi
+fi
+
+
+# Old stuff below this point
+
 if [ ! -f /usr/lib/ruby/vendor_ruby/gpgme.rb ]; then
     apt-get update
     apt-get -y install ruby-gpgme
 fi
 
-# this is useful to make the cmdline hiera tool work
-if [ -f /etc/hiera/data/secrets.yaml.asc -a ! -f /etc/hiera/data/secrets.yaml.gpg ]; then
-   (cd /etc/hiera/data && ln -s secrets.yaml.asc secrets.yaml.gpg)
-fi
-
-if [ ! -f /usr/bin/eyaml ]; then
-    apt-get update
-    apt-get -y install hiera-eyaml
-fi
-
-if [ ! -s $GNUPGHOME/secring.gpg -a ! -s $GNUPGHOME/pubring.kbx ]; then
+if [ ! -s $GNUPGHOME/secring.gpg ]; then
 
     if [ "x$1" != "x--force" ]; then
 	echo ""