diff --git a/global/overlay/etc/puppet/cosmos_enc.py b/global/overlay/etc/puppet/cosmos_enc.py index 131d161..852fb25 100755 --- a/global/overlay/etc/puppet/cosmos_enc.py +++ b/global/overlay/etc/puppet/cosmos_enc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys import yaml @@ -14,5 +14,5 @@ if os.path.exists(db_file): with open(db_file) as fd: db.update(yaml.load(fd)) -print yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[])))) +print(yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[]))))) diff --git a/global/post-tasks.d/015cosmos-trust b/global/post-tasks.d/015cosmos-trust index 74835e0..f9f4667 100755 --- a/global/post-tasks.d/015cosmos-trust +++ b/global/post-tasks.d/015cosmos-trust @@ -1,28 +1,78 @@ -#!/bin/sh +#!/bin/bash + +gnupg_show_options='--import --import-options show-only,import-minimal' +if [[ $(lsb_release -sr | awk -F . '{ print $1 }') -le 16 ]]; then + # gpg on Ubuntu 16 and less is gnupg < 2, which doesn't have --import-options show-only + # but on the other hand defaults to this mode (https://dev.gnupg.org/T2943) + gnupg_show_options='--dry-run' +fi if [ -z "$COSMOS_KEYS" ]; then COSMOS_KEYS=/etc/cosmos/keys fi -# Install new keys discovered in the $COSMOS_KEYS directory -for k in $COSMOS_KEYS/*.pub; do - fp=`cosmos gpg --with-colons --with-fingerprint < $k | awk -F: '$1 == "pub" {print $5}'` - fp_in_db=`cosmos gpg --with-colons --fingerprint | grep ":$fp:"` - if [ "x`echo $fp_in_db | grep '^pub:e:'`" != "x" ]; then - echo "$0: Key expired, will re-import it from $k" - cosmos gpg --fingerprint $fp - fi - # The removal of any ^pub:e: entrys means to ignore expired keys - thereby importing them again. - echo $fp_in_db | grep -v "^pub:e:" | grep -q ":$fp:" || cosmos gpg --import < $k +bold='\e[1m' +reset='\e[0m' +red='\033[01;31m' + +# Associative array of fingerprints in the GPG keyring +declare -A KEYRING + +# Associative array with expired keys in the GPG keyring +declare -A EXPIRED + +# associative array with non-expired keys found in $COSMOS_KEYS directory +declare -A SEEN + +# Load information about all keys present in the GPG keyring +for line in $(cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" { print $2 ":" $5 }'); do + IFS=':' read -r expired fp <<< $line + KEYRING[$fp]='1' + if [[ $expired == 'e' ]]; then + EXPIRED[$fp]=1 + fi done -# Delete keys no longer present in $COSMOS_KEYS directory -for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5}'`; do - seen="no" - for k in $COSMOS_KEYS/*.pub; do - cosmos gpg --with-colons --with-fingerprint < $k | grep -q ":$fp:" && seen="yes" - done - if [ "x$seen" = "xno" ]; then - cosmos gpg --yes --batch --delete-key $fp || true - fi +# Install new keys discovered in the $COSMOS_KEYS directory +for k in $COSMOS_KEYS/*.pub; do + if [[ ! -s $k ]]; then + # Silently ignore empty files + continue + fi + pubkeys_in_file=$(cosmos gpg ${gnupg_show_options} \ + --with-colons --with-fingerprint --quiet < $k \ + | grep "^pub:") + non_expired_pubkeys_in_file=$(echo ${pubkeys_in_file} | awk -F: '$2 != "e" { print $0 }') + if [[ ! $non_expired_pubkeys_in_file ]]; then + echo -e "$0: ${red}Ignoring file with expired pubkey: ${k}${reset}" + continue + fi + + fp=$(echo ${pubkeys_in_file} | awk -F: '{print $5}') + + # Remember that we saw fingerprint $fp in file $k + SEEN[$fp]=$k + + if [[ ! ${KEYRING[$fp]} ]]; then + echo -e "$0: ${bold}Importing new key ${fp}${reset} from ${k}" + cosmos gpg --import < $k + elif [[ ${EXPIRED[$fp]} ]]; then + echo -e "$0: ${bold}Re-importing expired key ${fp}${reset} from ${k}" + cosmos gpg --import < $k + fi +done + +if [[ ! ${#SEEN[@]} ]]; then + echo "$0: ${red}NO trusted keys found in directory ${COSMOS_KEYS} - aborting${reset}" + echo "(this is probably a syntax problem with the gpg commands in this script)" + exit 1 +fi + +# Delete keys no longer present (or expired) in $COSMOS_KEYS directory +for fp in ${!KEYRING[@]}; do + if [[ ! ${SEEN[$fp]} ]]; then + echo -e "$0: ${bold}Deleting key${reset} ${fp} not present (or expired) in ${COSMOS_KEYS}" + cosmos gpg --fingerprint $fp + cosmos gpg --yes --batch --delete-key $fp || true + fi done diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 9370e10..ee1889f 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -1,19 +1,24 @@ #!/bin/bash CONFIG=${CONFIG:=/etc/puppet/cosmos-modules.conf} +LOCALCONFIG=${LOCALCONFIG:=/etc/puppet/cosmos-modules_local.conf} CACHE_DIR=/var/cache/puppet-modules MODULES_DIR=${MODULES_DIR:=/etc/puppet/cosmos-modules} export GNUPGHOME=/etc/cosmos/gnupg -python -c "import yaml" 2>/dev/null || apt-get -y install python-yaml +# /etc/puppet/cosmos_enc.py needs the YAML module +python3 -c "import yaml" 2>/dev/null || apt-get -y install python3-yaml +bold='\e[1m' +reset='\e[0m' +red='\033[01;31m' stage_module() { rm -rf $CACHE_DIR/staging/$1 git archive --format=tar --prefix=$1/ $2 | (cd $CACHE_DIR/staging/ && tar xf -) } -if [ -f $CONFIG ]; then +if [ -f $CONFIG -o $LOCALCONFIG ]; then if [ ! -d $MODULES_DIR ]; then mkdir -p $MODULES_DIR fi @@ -21,11 +26,14 @@ if [ -f $CONFIG ]; then mkdir -p $CACHE_DIR/{scm,staging} fi + test -f $CONFIG || CONFIG='' + test -f $LOCALCONFIG || LOCALCONFIG='' + # First pass to clone any new modules, and update those marked for updating. - grep -E -v "^#" $CONFIG | ( + grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do - # We only support git:// urls and https:// urls atm - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then if [ ! -d $CACHE_DIR/scm/$module ]; then git clone -q $src $CACHE_DIR/scm/$module elif [ -d $CACHE_DIR/scm/$module/.git ]; then @@ -39,16 +47,14 @@ if [ -f $CONFIG ]; then continue fi else - echo "ERROR: Ignoring non-git repository" + echo -e "${red}ERROR: Ignoring non-git repository${reset}" continue fi elif [[ "$src" =~ .*:// ]]; then - echo "ERROR: Don't know how to install '$src'" + echo -e "${red}ERROR: Don't know how to install '${src}'${reset}" continue else - echo "WARNING" - echo "WARNING - attempting UNSAFE installation/upgrade of puppet-module $module from $src" - echo "WARNING" + echo -e "${bold}WARNING - attempting UNSAFE installation/upgrade of puppet-module ${module} from ${src}${reset}" if [ ! -d /etc/puppet/modules/$module ]; then puppet module install $src elif [ "$update" = "yes" ]; then @@ -60,34 +66,32 @@ if [ -f $CONFIG ]; then # Second pass to verify the signatures on all modules and stage those that # have good signatures. - grep -E -v "^#" $CONFIG | ( + grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do - # We only support git:// urls atm - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then # Verify git tag cd $CACHE_DIR/scm/$module TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) if [ "$COSMOS_VERBOSE" = "y" ]; then - echo "" - echo "Checking signature on tag ${TAG} for puppet-module $module" + echo -e "Checking signature on puppet-module:tag ${bold}${module}:${TAG}${reset}" fi if [ -z "$TAG" ]; then - echo "ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module $module" + echo -e "${red}ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module ${module}${reset}" continue fi git tag -v $TAG &> /dev/null if [ $? == 0 ]; then - if [ "$COSMOS_VERBOSE" = "y" ]; then - # short output on good signature - git tag -v $TAG 2>&1 | grep "gpg: Good signature" - fi + #if [ "$COSMOS_VERBOSE" = "y" ]; then + # # short output on good signature + # git tag -v $TAG 2>&1 | grep "gpg: Good signature" + #fi # Put archive in staging since tag verified OK stage_module $module $TAG else - echo "################################################################" - echo "FAILED signature check on puppet-module $module" - echo "################################################################" + echo -e "${red}FAILED signature check on puppet-module ${module}${reset}" git tag -v $TAG + echo '' fi fi done @@ -95,9 +99,9 @@ if [ -f $CONFIG ]; then # Cleanup removed puppet modules from CACHE_DIR for MODULE in $(ls -1 $CACHE_DIR/staging/); do - if ! grep -E -q "^$MODULE\s+" $CONFIG; then - rm -rf $CACHE_DIR/{scm,staging}/$MODULE - fi + if ! grep -h -E -q "^$MODULE\s+" $CONFIG $LOCALCONFIG; then + rm -rf $CACHE_DIR/{scm,staging}/$MODULE + fi done # Installing verified puppet modules diff --git a/global/post-tasks.d/020reports b/global/post-tasks.d/020reports index 091a236..c6033bb 100755 --- a/global/post-tasks.d/020reports +++ b/global/post-tasks.d/020reports @@ -1,4 +1,5 @@ #!/bin/sh -rm -f /var/run/facts.json -facter -p -y > /var/run/facts.yaml +#rm -f /var/run/facts.json +#facter -p -y > /var/run/facts.yaml +rm -f /var/run/facts.yaml diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet index b94b9ff..af45005 100755 --- a/global/post-tasks.d/030puppet +++ b/global/post-tasks.d/030puppet @@ -8,6 +8,7 @@ fi if [ -f /usr/bin/puppet -a -d /etc/puppet/manifests ]; then for m in `find /etc/puppet/manifests -name \*.pp`; do + test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" puppet apply $args $m done fi diff --git a/global/post-tasks.d/099autoremove b/global/post-tasks.d/099autoremove index 74b0aa4..c3c809c 100755 --- a/global/post-tasks.d/099autoremove +++ b/global/post-tasks.d/099autoremove @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +export DEBIAN_FRONTEND='noninteractive' if (( $RANDOM % 20 == 0)); then apt-get -qq update diff --git a/global/post-tasks.d/999reboot b/global/post-tasks.d/999reboot index 2ed9fa7..e0a05e1 100755 --- a/global/post-tasks.d/999reboot +++ b/global/post-tasks.d/999reboot @@ -1,5 +1,26 @@ -#!/bin/sh +#!/bin/bash -if [ -f /var/run/reboot-required -a -f /etc/cosmos-automatic-reboot ]; then - reboot +if [[ -f /var/run/reboot-required && -f /etc/cosmos-automatic-reboot ]]; then + + if [[ $HOSTNAME =~ -tug- ]]; then + # Reboot hosts in site TUG with 15 seconds delay (enough to manually + # cancel the reboot if logged in and seeing the 'emerg' message broadcasted to console) + sleep=15 + elif [[ $HOSTNAME =~ -fre- ]]; then + # reboot hosts in site FRE with 15+180 to 15+180+180 seconds delay + sleep=$(( 180 + ($RANDOM % 180))) + elif [[ $HOSTNAME =~ -lla- ]]; then + # reboot hosts in site LLA with 15+180+180 to 15+180+180+180 seconds delay + sleep=$(( 375 + ($RANDOM % 180))) + else + # reboot hosts in any other site with 15 to 315 seconds delay + sleep=$(( 15 + ($RANDOM % 300))) + fi + + logger -p local0.emerg -i -t cosmos-automatic-reboot "Rebooting automatically in $sleep seconds (if /var/run/reboot-required still exists)" + sleep $sleep + if [ -f /var/run/reboot-required ]; then + logger -p local0.crit -i -t cosmos-automatic-reboot "Rebooting automatically" + reboot + fi fi