From 761963ba2f0892096c708c6f880398a11ebc1956 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:06:48 +0100 Subject: [PATCH 1/9] add colors, sanity checking and support for Ubuntu 18.04 --- global/post-tasks.d/015cosmos-trust | 90 ++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/global/post-tasks.d/015cosmos-trust b/global/post-tasks.d/015cosmos-trust index 74835e0..f9f4667 100755 --- a/global/post-tasks.d/015cosmos-trust +++ b/global/post-tasks.d/015cosmos-trust @@ -1,28 +1,78 @@ -#!/bin/sh +#!/bin/bash + +gnupg_show_options='--import --import-options show-only,import-minimal' +if [[ $(lsb_release -sr | awk -F . '{ print $1 }') -le 16 ]]; then + # gpg on Ubuntu 16 and less is gnupg < 2, which doesn't have --import-options show-only + # but on the other hand defaults to this mode (https://dev.gnupg.org/T2943) + gnupg_show_options='--dry-run' +fi if [ -z "$COSMOS_KEYS" ]; then COSMOS_KEYS=/etc/cosmos/keys fi -# Install new keys discovered in the $COSMOS_KEYS directory -for k in $COSMOS_KEYS/*.pub; do - fp=`cosmos gpg --with-colons --with-fingerprint < $k | awk -F: '$1 == "pub" {print $5}'` - fp_in_db=`cosmos gpg --with-colons --fingerprint | grep ":$fp:"` - if [ "x`echo $fp_in_db | grep '^pub:e:'`" != "x" ]; then - echo "$0: Key expired, will re-import it from $k" - cosmos gpg --fingerprint $fp - fi - # The removal of any ^pub:e: entrys means to ignore expired keys - thereby importing them again. - echo $fp_in_db | grep -v "^pub:e:" | grep -q ":$fp:" || cosmos gpg --import < $k +bold='\e[1m' +reset='\e[0m' +red='\033[01;31m' + +# Associative array of fingerprints in the GPG keyring +declare -A KEYRING + +# Associative array with expired keys in the GPG keyring +declare -A EXPIRED + +# associative array with non-expired keys found in $COSMOS_KEYS directory +declare -A SEEN + +# Load information about all keys present in the GPG keyring +for line in $(cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" { print $2 ":" $5 }'); do + IFS=':' read -r expired fp <<< $line + KEYRING[$fp]='1' + if [[ $expired == 'e' ]]; then + EXPIRED[$fp]=1 + fi done -# Delete keys no longer present in $COSMOS_KEYS directory -for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5}'`; do - seen="no" - for k in $COSMOS_KEYS/*.pub; do - cosmos gpg --with-colons --with-fingerprint < $k | grep -q ":$fp:" && seen="yes" - done - if [ "x$seen" = "xno" ]; then - cosmos gpg --yes --batch --delete-key $fp || true - fi +# Install new keys discovered in the $COSMOS_KEYS directory +for k in $COSMOS_KEYS/*.pub; do + if [[ ! -s $k ]]; then + # Silently ignore empty files + continue + fi + pubkeys_in_file=$(cosmos gpg ${gnupg_show_options} \ + --with-colons --with-fingerprint --quiet < $k \ + | grep "^pub:") + non_expired_pubkeys_in_file=$(echo ${pubkeys_in_file} | awk -F: '$2 != "e" { print $0 }') + if [[ ! $non_expired_pubkeys_in_file ]]; then + echo -e "$0: ${red}Ignoring file with expired pubkey: ${k}${reset}" + continue + fi + + fp=$(echo ${pubkeys_in_file} | awk -F: '{print $5}') + + # Remember that we saw fingerprint $fp in file $k + SEEN[$fp]=$k + + if [[ ! ${KEYRING[$fp]} ]]; then + echo -e "$0: ${bold}Importing new key ${fp}${reset} from ${k}" + cosmos gpg --import < $k + elif [[ ${EXPIRED[$fp]} ]]; then + echo -e "$0: ${bold}Re-importing expired key ${fp}${reset} from ${k}" + cosmos gpg --import < $k + fi +done + +if [[ ! ${#SEEN[@]} ]]; then + echo "$0: ${red}NO trusted keys found in directory ${COSMOS_KEYS} - aborting${reset}" + echo "(this is probably a syntax problem with the gpg commands in this script)" + exit 1 +fi + +# Delete keys no longer present (or expired) in $COSMOS_KEYS directory +for fp in ${!KEYRING[@]}; do + if [[ ! ${SEEN[$fp]} ]]; then + echo -e "$0: ${bold}Deleting key${reset} ${fp} not present (or expired) in ${COSMOS_KEYS}" + cosmos gpg --fingerprint $fp + cosmos gpg --yes --batch --delete-key $fp || true + fi done From a2e4c5372f791ba963af18c790c282cfe8b93b7f Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:07:47 +0100 Subject: [PATCH 2/9] add support for a second, local puppet module config file --- global/post-tasks.d/018packages | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 9370e10..4e109d0 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -1,6 +1,7 @@ #!/bin/bash CONFIG=${CONFIG:=/etc/puppet/cosmos-modules.conf} +LOCALCONFIG=${LOCALCONFIG:=/etc/puppet/cosmos-modules_local.conf} CACHE_DIR=/var/cache/puppet-modules MODULES_DIR=${MODULES_DIR:=/etc/puppet/cosmos-modules} export GNUPGHOME=/etc/cosmos/gnupg @@ -13,7 +14,7 @@ stage_module() { git archive --format=tar --prefix=$1/ $2 | (cd $CACHE_DIR/staging/ && tar xf -) } -if [ -f $CONFIG ]; then +if [ -f $CONFIG -o $LOCALCONFIG ]; then if [ ! -d $MODULES_DIR ]; then mkdir -p $MODULES_DIR fi @@ -21,8 +22,11 @@ if [ -f $CONFIG ]; then mkdir -p $CACHE_DIR/{scm,staging} fi + test -f $CONFIG || CONFIG='' + test -f $LOCALCONFIG || LOCALCONFIG='' + # First pass to clone any new modules, and update those marked for updating. - grep -E -v "^#" $CONFIG | ( + grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do # We only support git:// urls and https:// urls atm if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then @@ -60,7 +64,7 @@ if [ -f $CONFIG ]; then # Second pass to verify the signatures on all modules and stage those that # have good signatures. - grep -E -v "^#" $CONFIG | ( + grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do # We only support git:// urls atm if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then @@ -95,9 +99,9 @@ if [ -f $CONFIG ]; then # Cleanup removed puppet modules from CACHE_DIR for MODULE in $(ls -1 $CACHE_DIR/staging/); do - if ! grep -E -q "^$MODULE\s+" $CONFIG; then - rm -rf $CACHE_DIR/{scm,staging}/$MODULE - fi + if ! grep -h -E -q "^$MODULE\s+" $CONFIG $LOCALCONFIG; then + rm -rf $CACHE_DIR/{scm,staging}/$MODULE + fi done # Installing verified puppet modules From bf1b476d9a5579c674ad2095f2d9c5481f4cb7d0 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:08:39 +0100 Subject: [PATCH 3/9] colors --- global/post-tasks.d/018packages | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 4e109d0..bd97488 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -8,6 +8,9 @@ export GNUPGHOME=/etc/cosmos/gnupg python -c "import yaml" 2>/dev/null || apt-get -y install python-yaml +bold='\e[1m' +reset='\e[0m' +red='\033[01;31m' stage_module() { rm -rf $CACHE_DIR/staging/$1 @@ -43,16 +46,14 @@ if [ -f $CONFIG -o $LOCALCONFIG ]; then continue fi else - echo "ERROR: Ignoring non-git repository" + echo -e "${red}ERROR: Ignoring non-git repository${reset}" continue fi elif [[ "$src" =~ .*:// ]]; then - echo "ERROR: Don't know how to install '$src'" + echo -e "${red}ERROR: Don't know how to install '${src}'${reset}" continue else - echo "WARNING" - echo "WARNING - attempting UNSAFE installation/upgrade of puppet-module $module from $src" - echo "WARNING" + echo -e "${bold}WARNING - attempting UNSAFE installation/upgrade of puppet-module ${module} from ${src}${reset}" if [ ! -d /etc/puppet/modules/$module ]; then puppet module install $src elif [ "$update" = "yes" ]; then @@ -72,26 +73,24 @@ if [ -f $CONFIG -o $LOCALCONFIG ]; then cd $CACHE_DIR/scm/$module TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) if [ "$COSMOS_VERBOSE" = "y" ]; then - echo "" - echo "Checking signature on tag ${TAG} for puppet-module $module" + echo -e "Checking signature on puppet-module:tag ${bold}${module}:${TAG}${reset}" fi if [ -z "$TAG" ]; then - echo "ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module $module" + echo -e "${red}ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module ${module}${reset}" continue fi git tag -v $TAG &> /dev/null if [ $? == 0 ]; then - if [ "$COSMOS_VERBOSE" = "y" ]; then - # short output on good signature - git tag -v $TAG 2>&1 | grep "gpg: Good signature" - fi + #if [ "$COSMOS_VERBOSE" = "y" ]; then + # # short output on good signature + # git tag -v $TAG 2>&1 | grep "gpg: Good signature" + #fi # Put archive in staging since tag verified OK stage_module $module $TAG else - echo "################################################################" - echo "FAILED signature check on puppet-module $module" - echo "################################################################" + echo -e "${red}FAILED signature check on puppet-module ${module}${reset}" git tag -v $TAG + echo '' fi fi done From 08979437b5d6cfdf769e76b60563f2231577aa05 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:09:24 +0100 Subject: [PATCH 4/9] add support for file:// urls --- global/post-tasks.d/018packages | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index bd97488..79c3348 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -31,8 +31,8 @@ if [ -f $CONFIG -o $LOCALCONFIG ]; then # First pass to clone any new modules, and update those marked for updating. grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do - # We only support git:// urls and https:// urls atm - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then if [ ! -d $CACHE_DIR/scm/$module ]; then git clone -q $src $CACHE_DIR/scm/$module elif [ -d $CACHE_DIR/scm/$module/.git ]; then @@ -67,8 +67,8 @@ if [ -f $CONFIG -o $LOCALCONFIG ]; then # have good signatures. grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do - # We only support git:// urls atm - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then # Verify git tag cd $CACHE_DIR/scm/$module TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) From e069bd4f0642726dbd6f91250ed8f989346e0fa3 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:10:46 +0100 Subject: [PATCH 5/9] remove unused reports that take about 2s per run to create --- global/post-tasks.d/020reports | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/global/post-tasks.d/020reports b/global/post-tasks.d/020reports index 091a236..c6033bb 100755 --- a/global/post-tasks.d/020reports +++ b/global/post-tasks.d/020reports @@ -1,4 +1,5 @@ #!/bin/sh -rm -f /var/run/facts.json -facter -p -y > /var/run/facts.yaml +#rm -f /var/run/facts.json +#facter -p -y > /var/run/facts.yaml +rm -f /var/run/facts.yaml From bc027359d10eaa9794f7c982b1ea398a2d876fb2 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:11:15 +0100 Subject: [PATCH 6/9] show which manifest is applied to lessen confusion --- global/post-tasks.d/030puppet | 1 + 1 file changed, 1 insertion(+) diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet index b94b9ff..af45005 100755 --- a/global/post-tasks.d/030puppet +++ b/global/post-tasks.d/030puppet @@ -8,6 +8,7 @@ fi if [ -f /usr/bin/puppet -a -d /etc/puppet/manifests ]; then for m in `find /etc/puppet/manifests -name \*.pp`; do + test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" puppet apply $args $m done fi From 5eeaa2e3ff8172b486522ccb7d24c623935f797b Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:12:07 +0100 Subject: [PATCH 7/9] noninteractive to not block when removing packages --- global/post-tasks.d/099autoremove | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/global/post-tasks.d/099autoremove b/global/post-tasks.d/099autoremove index 74b0aa4..c3c809c 100755 --- a/global/post-tasks.d/099autoremove +++ b/global/post-tasks.d/099autoremove @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +export DEBIAN_FRONTEND='noninteractive' if (( $RANDOM % 20 == 0)); then apt-get -qq update From fc3d3294ed3de90442bbc96d2b23f2b141e0f7f8 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:12:41 +0100 Subject: [PATCH 8/9] stage reboots across sites --- global/post-tasks.d/999reboot | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/global/post-tasks.d/999reboot b/global/post-tasks.d/999reboot index 2ed9fa7..e0a05e1 100755 --- a/global/post-tasks.d/999reboot +++ b/global/post-tasks.d/999reboot @@ -1,5 +1,26 @@ -#!/bin/sh +#!/bin/bash -if [ -f /var/run/reboot-required -a -f /etc/cosmos-automatic-reboot ]; then - reboot +if [[ -f /var/run/reboot-required && -f /etc/cosmos-automatic-reboot ]]; then + + if [[ $HOSTNAME =~ -tug- ]]; then + # Reboot hosts in site TUG with 15 seconds delay (enough to manually + # cancel the reboot if logged in and seeing the 'emerg' message broadcasted to console) + sleep=15 + elif [[ $HOSTNAME =~ -fre- ]]; then + # reboot hosts in site FRE with 15+180 to 15+180+180 seconds delay + sleep=$(( 180 + ($RANDOM % 180))) + elif [[ $HOSTNAME =~ -lla- ]]; then + # reboot hosts in site LLA with 15+180+180 to 15+180+180+180 seconds delay + sleep=$(( 375 + ($RANDOM % 180))) + else + # reboot hosts in any other site with 15 to 315 seconds delay + sleep=$(( 15 + ($RANDOM % 300))) + fi + + logger -p local0.emerg -i -t cosmos-automatic-reboot "Rebooting automatically in $sleep seconds (if /var/run/reboot-required still exists)" + sleep $sleep + if [ -f /var/run/reboot-required ]; then + logger -p local0.crit -i -t cosmos-automatic-reboot "Rebooting automatically" + reboot + fi fi From f25a6af71280de14e815d48ce5d4eb84e42e4072 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 15 Jan 2019 13:18:22 +0100 Subject: [PATCH 9/9] use python3 --- global/overlay/etc/puppet/cosmos_enc.py | 4 ++-- global/post-tasks.d/018packages | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/global/overlay/etc/puppet/cosmos_enc.py b/global/overlay/etc/puppet/cosmos_enc.py index 131d161..852fb25 100755 --- a/global/overlay/etc/puppet/cosmos_enc.py +++ b/global/overlay/etc/puppet/cosmos_enc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys import yaml @@ -14,5 +14,5 @@ if os.path.exists(db_file): with open(db_file) as fd: db.update(yaml.load(fd)) -print yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[])))) +print(yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[]))))) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 79c3348..ee1889f 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -6,7 +6,8 @@ CACHE_DIR=/var/cache/puppet-modules MODULES_DIR=${MODULES_DIR:=/etc/puppet/cosmos-modules} export GNUPGHOME=/etc/cosmos/gnupg -python -c "import yaml" 2>/dev/null || apt-get -y install python-yaml +# /etc/puppet/cosmos_enc.py needs the YAML module +python3 -c "import yaml" 2>/dev/null || apt-get -y install python3-yaml bold='\e[1m' reset='\e[0m'