From b72030ac92e356ab1cd6ad30210c66a48ceb14bf Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Thu, 3 May 2018 12:48:05 +0200 Subject: [PATCH] merge goodies from eduid-cosmos --- global/post-tasks.d/015cosmos-trust | 75 +++++++++++++++++++++-------- global/post-tasks.d/018packages | 53 ++++++++++---------- global/post-tasks.d/030puppet | 5 +- global/post-tasks.d/099autoremove | 8 +-- global/post-tasks.d/999reboot | 27 +++++++++-- global/pre-tasks.d/030puppet | 18 ++++--- global/pre-tasks.d/040hiera-gpg | 7 ++- 7 files changed, 131 insertions(+), 62 deletions(-) diff --git a/global/post-tasks.d/015cosmos-trust b/global/post-tasks.d/015cosmos-trust index 74835e06..cbb748f9 100755 --- a/global/post-tasks.d/015cosmos-trust +++ b/global/post-tasks.d/015cosmos-trust @@ -1,28 +1,63 @@ -#!/bin/sh +#!/bin/bash if [ -z "$COSMOS_KEYS" ]; then COSMOS_KEYS=/etc/cosmos/keys fi -# Install new keys discovered in the $COSMOS_KEYS directory -for k in $COSMOS_KEYS/*.pub; do - fp=`cosmos gpg --with-colons --with-fingerprint < $k | awk -F: '$1 == "pub" {print $5}'` - fp_in_db=`cosmos gpg --with-colons --fingerprint | grep ":$fp:"` - if [ "x`echo $fp_in_db | grep '^pub:e:'`" != "x" ]; then - echo "$0: Key expired, will re-import it from $k" - cosmos gpg --fingerprint $fp - fi - # The removal of any ^pub:e: entrys means to ignore expired keys - thereby importing them again. - echo $fp_in_db | grep -v "^pub:e:" | grep -q ":$fp:" || cosmos gpg --import < $k +bold='\e[1m' +reset='\e[0m' +red='\033[01;31m' + +# Associative array of fingerprints in the GPG keyring +declare -A KEYRING + +# Associative array with expired keys in the GPG keyring +declare -A EXPIRED + +# associative array with non-expired keys found in $COSMOS_KEYS directory +declare -A SEEN + +# Load information about all keys present in the GPG keyring +for line in $(cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" { print $2 ":" $5 }'); do + IFS=':' read -r expired fp <<< $line + KEYRING[$fp]='1' + if [[ $expired == 'e' ]]; then + EXPIRED[$fp]=1 + fi done -# Delete keys no longer present in $COSMOS_KEYS directory -for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5}'`; do - seen="no" - for k in $COSMOS_KEYS/*.pub; do - cosmos gpg --with-colons --with-fingerprint < $k | grep -q ":$fp:" && seen="yes" - done - if [ "x$seen" = "xno" ]; then - cosmos gpg --yes --batch --delete-key $fp || true - fi +# Install new keys discovered in the $COSMOS_KEYS directory +for k in $COSMOS_KEYS/*.pub; do + if [[ ! -s $k ]]; then + # Silently ignore empty files + continue + fi + pubkeys_in_file=$(cosmos gpg --with-colons --with-fingerprint < $k | grep "^pub:") + non_expired_pubkeys_in_file=$(echo ${pubkeys_in_file} | awk -F: '$2 != "e" { print $0 }') + if [[ ! $non_expired_pubkeys_in_file ]]; then + echo -e "$0: ${red}Ignoring file with expired pubkey: ${k}${reset}" + continue + fi + + fp=$(echo ${pubkeys_in_file} | awk -F: '{print $5}') + + # Remember that we saw fingerprint $fp in file $k + SEEN[$fp]=$k + + if [[ ! ${KEYRING[$fp]} ]]; then + echo -e "$0: ${bold}Importing new key ${fp}${reset} from ${k}" + cosmos gpg --import < $k + elif [[ ${EXPIRED[$fp]} ]]; then + echo -e "$0: ${bold}Re-importing expired key ${fp}${reset} from ${k}" + cosmos gpg --import < $k + fi +done + +# Delete keys no longer present (or expired) in $COSMOS_KEYS directory +for fp in ${!KEYRING[@]}; do + if [[ ! ${SEEN[$fp]} ]]; then + echo -e "$0: ${bold}Deleting key${reset} ${fp} not present (or expired) in ${COSMOS_KEYS}" + cosmos gpg --fingerprint $fp + cosmos gpg --yes --batch --delete-key $fp || true + fi done diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 9370e102..79c33483 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -1,19 +1,23 @@ #!/bin/bash CONFIG=${CONFIG:=/etc/puppet/cosmos-modules.conf} +LOCALCONFIG=${LOCALCONFIG:=/etc/puppet/cosmos-modules_local.conf} CACHE_DIR=/var/cache/puppet-modules MODULES_DIR=${MODULES_DIR:=/etc/puppet/cosmos-modules} export GNUPGHOME=/etc/cosmos/gnupg python -c "import yaml" 2>/dev/null || apt-get -y install python-yaml +bold='\e[1m' +reset='\e[0m' +red='\033[01;31m' stage_module() { rm -rf $CACHE_DIR/staging/$1 git archive --format=tar --prefix=$1/ $2 | (cd $CACHE_DIR/staging/ && tar xf -) } -if [ -f $CONFIG ]; then +if [ -f $CONFIG -o $LOCALCONFIG ]; then if [ ! -d $MODULES_DIR ]; then mkdir -p $MODULES_DIR fi @@ -21,11 +25,14 @@ if [ -f $CONFIG ]; then mkdir -p $CACHE_DIR/{scm,staging} fi + test -f $CONFIG || CONFIG='' + test -f $LOCALCONFIG || LOCALCONFIG='' + # First pass to clone any new modules, and update those marked for updating. - grep -E -v "^#" $CONFIG | ( + grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do - # We only support git:// urls and https:// urls atm - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then if [ ! -d $CACHE_DIR/scm/$module ]; then git clone -q $src $CACHE_DIR/scm/$module elif [ -d $CACHE_DIR/scm/$module/.git ]; then @@ -39,16 +46,14 @@ if [ -f $CONFIG ]; then continue fi else - echo "ERROR: Ignoring non-git repository" + echo -e "${red}ERROR: Ignoring non-git repository${reset}" continue fi elif [[ "$src" =~ .*:// ]]; then - echo "ERROR: Don't know how to install '$src'" + echo -e "${red}ERROR: Don't know how to install '${src}'${reset}" continue else - echo "WARNING" - echo "WARNING - attempting UNSAFE installation/upgrade of puppet-module $module from $src" - echo "WARNING" + echo -e "${bold}WARNING - attempting UNSAFE installation/upgrade of puppet-module ${module} from ${src}${reset}" if [ ! -d /etc/puppet/modules/$module ]; then puppet module install $src elif [ "$update" = "yes" ]; then @@ -60,34 +65,32 @@ if [ -f $CONFIG ]; then # Second pass to verify the signatures on all modules and stage those that # have good signatures. - grep -E -v "^#" $CONFIG | ( + grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | ( while read module src update pattern; do - # We only support git:// urls atm - if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then + # We only support git://, file:/// and https:// urls at the moment + if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then # Verify git tag cd $CACHE_DIR/scm/$module TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) if [ "$COSMOS_VERBOSE" = "y" ]; then - echo "" - echo "Checking signature on tag ${TAG} for puppet-module $module" + echo -e "Checking signature on puppet-module:tag ${bold}${module}:${TAG}${reset}" fi if [ -z "$TAG" ]; then - echo "ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module $module" + echo -e "${red}ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module ${module}${reset}" continue fi git tag -v $TAG &> /dev/null if [ $? == 0 ]; then - if [ "$COSMOS_VERBOSE" = "y" ]; then - # short output on good signature - git tag -v $TAG 2>&1 | grep "gpg: Good signature" - fi + #if [ "$COSMOS_VERBOSE" = "y" ]; then + # # short output on good signature + # git tag -v $TAG 2>&1 | grep "gpg: Good signature" + #fi # Put archive in staging since tag verified OK stage_module $module $TAG else - echo "################################################################" - echo "FAILED signature check on puppet-module $module" - echo "################################################################" + echo -e "${red}FAILED signature check on puppet-module ${module}${reset}" git tag -v $TAG + echo '' fi fi done @@ -95,9 +98,9 @@ if [ -f $CONFIG ]; then # Cleanup removed puppet modules from CACHE_DIR for MODULE in $(ls -1 $CACHE_DIR/staging/); do - if ! grep -E -q "^$MODULE\s+" $CONFIG; then - rm -rf $CACHE_DIR/{scm,staging}/$MODULE - fi + if ! grep -h -E -q "^$MODULE\s+" $CONFIG $LOCALCONFIG; then + rm -rf $CACHE_DIR/{scm,staging}/$MODULE + fi done # Installing verified puppet modules diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet index 67429497..af450057 100755 --- a/global/post-tasks.d/030puppet +++ b/global/post-tasks.d/030puppet @@ -1,13 +1,14 @@ #!/bin/sh if [ "x$COSMOS_VERBOSE" = "xy" ]; then - args="--verbose" + args="--verbose --show_diff" else args="--logdest=syslog" fi if [ -f /usr/bin/puppet -a -d /etc/puppet/manifests ]; then for m in `find /etc/puppet/manifests -name \*.pp`; do - puppet apply $args < $m + test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" + puppet apply $args $m done fi diff --git a/global/post-tasks.d/099autoremove b/global/post-tasks.d/099autoremove index 2cc69968..9911ae2f 100755 --- a/global/post-tasks.d/099autoremove +++ b/global/post-tasks.d/099autoremove @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash -apt-get -qq update -apt-get -qq -y autoremove +if (( $RANDOM % 20 == 0)); then + apt-get -qq update + apt-get -qq -y autoremove +fi diff --git a/global/post-tasks.d/999reboot b/global/post-tasks.d/999reboot index 2ed9fa7a..bc27e6ef 100755 --- a/global/post-tasks.d/999reboot +++ b/global/post-tasks.d/999reboot @@ -1,5 +1,26 @@ -#!/bin/sh +#!/bin/bash -if [ -f /var/run/reboot-required -a -f /etc/cosmos-automatic-reboot ]; then - reboot +if [[ -f /var/run/reboot-required && -f /etc/cosmos-automatic-reboot ]]; then + + if [[ $HOSTNAME =~ -tug- ]]; then + # Reboot hosts in site TUG with 15 seconds delay (enough to manually + # cancel the reboot if logged in and seeind the 'emerg' message broadcasted to console) + sleep=15 + elif [[ $HOSTNAME =~ -fre- ]]; then + # reboot hosts in site FRE with 15+180 to 15+180+180 seconds delay + sleep=$(( 180 + ($RANDOM % 180))) + elif [[ $HOSTNAME =~ -lla- ]]; then + # reboot hosts in site LLA with 15+180+180 to 15+180+180+180 seconds delay + sleep=$(( 375 + ($RANDOM % 180))) + else + # reboot hosts in any other site with 15 to 315 seconds delay + sleep=$(( 15 + ($RANDOM % 300))) + fi + + logger -p local0.emerg -i -t cosmos-automatic-reboot "Rebooting automatically in $sleep seconds (if /var/run/reboot-required still exists)" + sleep $sleep + if [ -f /var/run/reboot-required ]; then + logger -p local0.crit -i -t cosmos-automatic-reboot "Rebooting automatically" + reboot + fi fi diff --git a/global/pre-tasks.d/030puppet b/global/pre-tasks.d/030puppet index ef080161..4d8814ae 100755 --- a/global/pre-tasks.d/030puppet +++ b/global/pre-tasks.d/030puppet @@ -8,14 +8,16 @@ set -e stamp="$COSMOS_BASE/stamps/puppet-tools-v01.stamp" if ! test -f $stamp -a -f /usr/bin/puppet; then - codename=`lsb_release -c| awk '{print $2}'` - puppetdeb="$COSMOS_REPO/apt/puppetlabs-release-${codename}.deb" - if [ ! -f $puppetdeb ]; then - echo "$0: Puppet deb for release $codename not found in $COSMOS_REPO/apt/" - echo " Get it from https://apt.puppetlabs.com/ and put it in the Cosmos repo." - exit 1 - fi - dpkg -i $puppetdeb + #codename=`lsb_release -c| awk '{print $2}'` + #puppetdeb="$COSMOS_REPO/apt/puppetlabs-release-${codename}.deb" + #if [ ! -f $puppetdeb ]; then + # echo "$0: Puppet deb for release $codename not found in $COSMOS_REPO/apt/" + # echo " Get it from https://apt.puppetlabs.com/ and put it in the Cosmos repo." + # exit 1 + #fi + ## The key currently in use does not appear to actually be installed with $puppetdeb + #test -f apt-key add $COSMOS_REPO/apt/keys/puppetlabs-EF8D349F.pub && apt-key add $COSMOS_REPO/apt/keys/puppetlabs-EF8D349F.pub + #dpkg -i $puppetdeb apt-get update apt-get -y install puppet-common diff --git a/global/pre-tasks.d/040hiera-gpg b/global/pre-tasks.d/040hiera-gpg index 0ef2d86b..aed6dbe9 100755 --- a/global/pre-tasks.d/040hiera-gpg +++ b/global/pre-tasks.d/040hiera-gpg @@ -19,7 +19,12 @@ if [ -f /etc/hiera/data/secrets.yaml.asc -a ! -f /etc/hiera/data/secrets.yaml.gp (cd /etc/hiera/data && ln -s secrets.yaml.asc secrets.yaml.gpg) fi -if [ ! -s $GNUPGHOME/secring.gpg -a ! -s /etc/hiera/gpg/pubring.kbx ]; then +if [ ! -f /usr/bin/eyaml ]; then + apt-get update + apt-get -y install hiera-eyaml +fi + +if [ ! -s $GNUPGHOME/secring.gpg -a ! -s $GNUPGHOME/pubring.kbx ]; then if [ "x$1" != "x--force" ]; then echo ""