Merge pull request #6 from SUNET/ft-sync_from_eduid

sync from eduid

global/overlay/etc/puppet/cosmos_enc.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys
 import yaml
@@ -14,5 +14,5 @@ if os.path.exists(db_file):
    with open(db_file) as fd:
       db.update(yaml.load(fd))
 
-print yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[]))))
+print(yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[])))))
 

global/post-tasks.d/015cosmos-trust

@@ -1,28 +1,78 @@
-#!/bin/sh
+#!/bin/bash
+
+gnupg_show_options='--import --import-options show-only,import-minimal'
+if [[ $(lsb_release -sr | awk -F . '{ print $1 }') -le 16 ]]; then
+    # gpg on Ubuntu 16 and less is gnupg < 2, which doesn't have --import-options show-only
+    # but on the other hand defaults to this mode (https://dev.gnupg.org/T2943)
+    gnupg_show_options='--dry-run'
+fi
 
 if [ -z "$COSMOS_KEYS" ]; then
    COSMOS_KEYS=/etc/cosmos/keys
 fi
 
-# Install new keys discovered in the $COSMOS_KEYS directory
+bold='\e[1m'
-for k in $COSMOS_KEYS/*.pub; do
+reset='\e[0m'
-   fp=`cosmos gpg --with-colons --with-fingerprint < $k | awk -F: '$1 == "pub" {print $5}'`
+red='\033[01;31m'
-   fp_in_db=`cosmos gpg --with-colons --fingerprint | grep ":$fp:"`
+
-   if [ "x`echo $fp_in_db | grep '^pub:e:'`" != "x" ]; then
+# Associative array of fingerprints in the GPG keyring
-       echo "$0: Key expired, will re-import it from $k"
+declare -A KEYRING
-       cosmos gpg --fingerprint $fp
+
+# Associative array with expired keys in the GPG keyring
+declare -A EXPIRED
+
+# associative array with non-expired keys found in $COSMOS_KEYS directory
+declare -A SEEN
+
+# Load information about all keys present in the GPG keyring
+for line in $(cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" { print $2 ":" $5 }'); do
+    IFS=':' read -r expired fp <<< $line
+    KEYRING[$fp]='1'
+    if [[ $expired == 'e' ]]; then
+	EXPIRED[$fp]=1
     fi
-   # The removal of any ^pub:e: entrys means to ignore expired keys - thereby importing them again.
-   echo $fp_in_db | grep -v "^pub:e:" | grep -q ":$fp:" || cosmos gpg --import < $k
 done
 
-# Delete keys no longer present in $COSMOS_KEYS directory
+# Install new keys discovered in the $COSMOS_KEYS directory
-for fp in `cosmos gpg --with-colons --fingerprint | awk -F: '$1 == "pub" {print $5}'`; do
-   seen="no"
 for k in $COSMOS_KEYS/*.pub; do
-      cosmos gpg --with-colons --with-fingerprint < $k | grep -q ":$fp:" && seen="yes"
+    if [[ ! -s $k ]]; then
+	# Silently ignore empty files
+	continue
+    fi
+    pubkeys_in_file=$(cosmos gpg ${gnupg_show_options} \
+			     --with-colons --with-fingerprint --quiet < $k \
+			  | grep "^pub:")
+    non_expired_pubkeys_in_file=$(echo ${pubkeys_in_file} | awk -F: '$2 != "e" { print $0 }')
+    if [[ ! $non_expired_pubkeys_in_file ]]; then
+	echo -e "$0: ${red}Ignoring file with expired pubkey: ${k}${reset}"
+	continue
+    fi
+
+    fp=$(echo ${pubkeys_in_file} | awk -F: '{print $5}')
+
+    # Remember that we saw fingerprint $fp in file $k
+    SEEN[$fp]=$k
+
+    if [[ ! ${KEYRING[$fp]} ]]; then
+	echo -e "$0: ${bold}Importing new key ${fp}${reset} from ${k}"
+	cosmos gpg --import < $k
+    elif [[ ${EXPIRED[$fp]} ]]; then
+	echo -e "$0: ${bold}Re-importing expired key ${fp}${reset} from ${k}"
+	cosmos gpg --import < $k
+    fi
 done
-   if [ "x$seen" = "xno" ]; then
+
+if [[ ! ${#SEEN[@]} ]]; then
+    echo "$0: ${red}NO trusted keys found in directory ${COSMOS_KEYS} - aborting${reset}"
+    echo "(this is probably a syntax problem with the gpg commands in this script)"
+    exit 1
+fi
+
+# Delete keys no longer present (or expired) in $COSMOS_KEYS directory
+for fp in ${!KEYRING[@]}; do
+    if [[ ! ${SEEN[$fp]} ]]; then
+	echo -e "$0: ${bold}Deleting key${reset} ${fp} not present (or expired) in ${COSMOS_KEYS}"
+	cosmos gpg --fingerprint $fp
 	cosmos gpg --yes --batch --delete-key $fp || true
     fi
 done

global/post-tasks.d/018packages

@@ -1,19 +1,24 @@
 #!/bin/bash
 
 CONFIG=${CONFIG:=/etc/puppet/cosmos-modules.conf}
+LOCALCONFIG=${LOCALCONFIG:=/etc/puppet/cosmos-modules_local.conf}
 CACHE_DIR=/var/cache/puppet-modules
 MODULES_DIR=${MODULES_DIR:=/etc/puppet/cosmos-modules}
 export GNUPGHOME=/etc/cosmos/gnupg
 
-python -c "import yaml" 2>/dev/null || apt-get -y install python-yaml
+# /etc/puppet/cosmos_enc.py needs the YAML module
+python3 -c "import yaml" 2>/dev/null || apt-get -y install python3-yaml
 
+bold='\e[1m'
+reset='\e[0m'
+red='\033[01;31m'
 
 stage_module() {
   rm -rf $CACHE_DIR/staging/$1
   git archive --format=tar --prefix=$1/ $2 | (cd $CACHE_DIR/staging/ && tar xf -)
 }
 
-if [ -f $CONFIG ]; then
+if [ -f $CONFIG -o $LOCALCONFIG ]; then
   if [ ! -d $MODULES_DIR ]; then
     mkdir -p $MODULES_DIR
   fi
@@ -21,11 +26,14 @@ if [ -f $CONFIG ]; then
     mkdir -p $CACHE_DIR/{scm,staging}
   fi
 
+  test -f $CONFIG || CONFIG=''
+  test -f $LOCALCONFIG || LOCALCONFIG=''
+
   # First pass to clone any new modules, and update those marked for updating.
-  grep -E -v "^#" $CONFIG | (
+  grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | (
     while read module src update pattern; do
-      # We only support git:// urls and https:// urls atm
+      # We only support git://, file:/// and https:// urls at the moment
-      if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then
+      if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then
         if [ ! -d $CACHE_DIR/scm/$module ]; then
           git clone -q $src $CACHE_DIR/scm/$module
         elif [ -d $CACHE_DIR/scm/$module/.git ]; then
@@ -39,16 +47,14 @@ if [ -f $CONFIG ]; then
             continue
           fi
         else
-          echo "ERROR: Ignoring non-git repository"
+          echo -e "${red}ERROR: Ignoring non-git repository${reset}"
           continue
         fi
       elif [[ "$src" =~ .*:// ]]; then
-        echo "ERROR: Don't know how to install '$src'"
+        echo -e "${red}ERROR: Don't know how to install '${src}'${reset}"
         continue
       else
-        echo "WARNING"
+        echo -e "${bold}WARNING - attempting UNSAFE installation/upgrade of puppet-module ${module} from ${src}${reset}"
-        echo "WARNING - attempting UNSAFE installation/upgrade of puppet-module $module from $src"
-        echo "WARNING"
         if [ ! -d /etc/puppet/modules/$module ]; then
           puppet module install $src
         elif [ "$update" = "yes" ]; then
@@ -60,34 +66,32 @@ if [ -f $CONFIG ]; then
 
   # Second pass to verify the signatures on all modules and stage those that
   # have good signatures.
-  grep -E -v "^#" $CONFIG | (
+  grep -h -E -v "^#" $CONFIG $LOCALCONFIG | sort | (
     while read module src update pattern; do
-      # We only support git:// urls atm
+      # We only support git://, file:/// and https:// urls at the moment
-      if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "https://" ]; then
+      if [ "${src:0:6}" = "git://" -o "${src:0:8}" = "file:///" -o "${src:0:8}" = "https://" ]; then
         # Verify git tag
         cd $CACHE_DIR/scm/$module
         TAG=$(git tag -l "${pattern:-*}" | sort | tail -1)
         if [ "$COSMOS_VERBOSE" = "y" ]; then
-          echo ""
+          echo -e "Checking signature on puppet-module:tag ${bold}${module}:${TAG}${reset}"
-          echo "Checking signature on tag ${TAG} for puppet-module $module"
         fi
         if [ -z "$TAG" ]; then
-          echo "ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module $module"
+          echo -e "${red}ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module ${module}${reset}"
           continue
         fi
         git tag -v $TAG &> /dev/null
         if [ $? == 0 ]; then
-          if [ "$COSMOS_VERBOSE" = "y" ]; then
+          #if [ "$COSMOS_VERBOSE" = "y" ]; then
-            # short output on good signature
+          #  # short output on good signature
-            git tag -v $TAG 2>&1 | grep "gpg: Good signature"
+          #  git tag -v $TAG 2>&1 | grep "gpg: Good signature"
-          fi
+          #fi
           # Put archive in staging since tag verified OK
           stage_module $module $TAG
         else
-          echo "################################################################"
+          echo -e "${red}FAILED signature check on puppet-module ${module}${reset}"
-          echo "FAILED signature check on puppet-module $module"
-          echo "################################################################"
           git tag -v $TAG
+	  echo ''
         fi
       fi
     done
@@ -95,7 +99,7 @@ if [ -f $CONFIG ]; then
 
   # Cleanup removed puppet modules from CACHE_DIR
   for MODULE in $(ls -1 $CACHE_DIR/staging/); do
-      if ! grep -E -q "^$MODULE\s+" $CONFIG; then
+    if ! grep -h -E -q "^$MODULE\s+" $CONFIG $LOCALCONFIG; then
       rm -rf $CACHE_DIR/{scm,staging}/$MODULE
     fi
   done

global/post-tasks.d/020reports

@@ -1,4 +1,5 @@
 #!/bin/sh
 
-rm -f /var/run/facts.json
+#rm -f /var/run/facts.json
-facter -p -y > /var/run/facts.yaml
+#facter -p -y > /var/run/facts.yaml
+rm -f /var/run/facts.yaml

global/post-tasks.d/030puppet

@@ -8,6 +8,7 @@ fi
 
 if [ -f /usr/bin/puppet -a -d /etc/puppet/manifests ]; then
    for m in `find /etc/puppet/manifests -name \*.pp`; do
+      test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m"
       puppet apply $args $m
    done
 fi

global/post-tasks.d/099autoremove

@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+export DEBIAN_FRONTEND='noninteractive'
 
 if (( $RANDOM % 20 == 0)); then
     apt-get -qq update

global/post-tasks.d/999reboot

@@ -1,5 +1,26 @@
-#!/bin/sh
+#!/bin/bash
 
-if [ -f /var/run/reboot-required -a -f /etc/cosmos-automatic-reboot ]; then
+if [[ -f /var/run/reboot-required && -f /etc/cosmos-automatic-reboot ]]; then
+
+    if [[ $HOSTNAME =~ -tug- ]]; then
+	# Reboot hosts in site TUG with 15 seconds delay (enough to manually
+	# cancel the reboot if logged in and seeing the 'emerg' message broadcasted to console)
+	sleep=15
+    elif [[ $HOSTNAME =~ -fre- ]]; then
+	# reboot hosts in site FRE with 15+180 to 15+180+180 seconds delay
+	sleep=$(( 180 + ($RANDOM % 180)))
+    elif [[ $HOSTNAME =~ -lla- ]]; then
+	# reboot hosts in site LLA with 15+180+180 to 15+180+180+180 seconds delay
+	sleep=$(( 375 + ($RANDOM % 180)))
+    else
+	# reboot hosts in any other site with 15 to 315 seconds delay
+	sleep=$(( 15 + ($RANDOM % 300)))
+    fi
+
+    logger -p local0.emerg -i -t cosmos-automatic-reboot "Rebooting automatically in $sleep seconds (if /var/run/reboot-required still exists)"
+    sleep $sleep
+    if [ -f /var/run/reboot-required ]; then
+	logger -p local0.crit -i -t cosmos-automatic-reboot "Rebooting automatically"
 	reboot
     fi
+fi