From ca353ed406bc46f09b168eb4be25edd2c9594cd7 Mon Sep 17 00:00:00 2001
From: John Van de Meulebrouck Brendgard <john@sunet.se>
Date: Fri, 17 Nov 2023 13:50:02 +0100
Subject: [PATCH 01/15] Set same permissions for /root/.ssh/authorized_keys

in post-tasks.d/010fix-ssh-perms as is done by
Puppet with sunet::ssh_keys.
---
 global/post-tasks.d/010fix-ssh-perms | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/global/post-tasks.d/010fix-ssh-perms b/global/post-tasks.d/010fix-ssh-perms
index 87636d79..f32a6a44 100755
--- a/global/post-tasks.d/010fix-ssh-perms
+++ b/global/post-tasks.d/010fix-ssh-perms
@@ -17,7 +17,7 @@ if test -f /root/.ssh/authorized_keys; then
     if test `stat -t /root/.ssh/authorized_keys | cut -d\  -f5` != 0; then
 	chown root.root /root/.ssh/authorized_keys
     fi
-    if test `stat --printf=%a /root/.ssh/authorized_keys` != 600; then
-	chmod 600 /root/.ssh/authorized_keys
+    if test `stat --printf=%a /root/.ssh/authorized_keys` != 440; then
+	chmod 440 /root/.ssh/authorized_keys
     fi
 fi

From 75e566ab61fe9d485a70f3066084705c720315b2 Mon Sep 17 00:00:00 2001
From: John Van de Meulebrouck Brendgard <john@sunet.se>
Date: Fri, 17 Nov 2023 14:58:51 +0100
Subject: [PATCH 02/15] Make sure that /root in overlay is owned by root

as well as that /root/.ssh and its content is
only owned and readable by root. This is redundant
if the previous permissions were properly applied
and no other changes have been made by the user
or something else, but is added for good measure
as a layered defense.
---
 global/pre-tasks.d/015set-overlay-permissions | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/global/pre-tasks.d/015set-overlay-permissions b/global/pre-tasks.d/015set-overlay-permissions
index 37f98441..205180b9 100755
--- a/global/pre-tasks.d/015set-overlay-permissions
+++ b/global/pre-tasks.d/015set-overlay-permissions
@@ -14,10 +14,17 @@ if ! test -d "$MODEL_OVERLAY"; then
     exit 0
 fi
 
+args=""
+if [ "x$COSMOS_VERBOSE" = "xy" ]; then
+    args="-v"
+fi
+
 if [ -d "$MODEL_OVERLAY/root" ]; then
-    args=""
-    if [ "x$COSMOS_VERBOSE" = "xy" ]; then
-        args="-v"
-    fi
+    chown ${args} root:root "$MODEL_OVERLAY"/root
     chmod ${args} 0700 "$MODEL_OVERLAY"/root
 fi
+
+if [ -d "$MODEL_OVERLAY/root/.ssh" ]; then
+    chown ${args} -R root:root "$MODEL_OVERLAY"/root/.ssh
+    chmod ${args} 0700 "$MODEL_OVERLAY"/root/.ssh
+fi

From 8d4ce2d1b7132dbe00486134e60e0e13f715b8d4 Mon Sep 17 00:00:00 2001
From: John Van de Meulebrouck Brendgard <john@sunet.se>
Date: Fri, 17 Nov 2023 15:03:47 +0100
Subject: [PATCH 03/15] Make sure that COSMOS_BASE is only readable

by root since it's possible that the directory
can contain files that after applying the
overlay to / only should be read or writable
by root.
---
 global/post-tasks.d/014set-cosmos-permissions | 24 +++++++++++++++++++
 global/pre-tasks.d/014set-cosmos-permissions  | 24 +++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100755 global/post-tasks.d/014set-cosmos-permissions
 create mode 100755 global/pre-tasks.d/014set-cosmos-permissions

diff --git a/global/post-tasks.d/014set-cosmos-permissions b/global/post-tasks.d/014set-cosmos-permissions
new file mode 100755
index 00000000..08992b5f
--- /dev/null
+++ b/global/post-tasks.d/014set-cosmos-permissions
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# Set Cosmos directory permissions so that
+# the files cannot be read by anyone but root,
+# since it's possible that the directory
+# can contain files that after applying the
+# overlay to / only should be read or writable
+# by root.
+
+set -e
+self=$(basename "$0")
+
+if ! test -d "$COSMOS_BASE"; then
+    test -z "$COSMOS_VERBOSE" || echo "$self: COSMOS_BASE was not found. Aborting change of permissions."
+    exit 0
+fi
+
+args=""
+if [ "x$COSMOS_VERBOSE" = "xy" ]; then
+    args="-v"
+fi
+
+chown ${args} root:root "$COSMOS_BASE"
+chmod ${args} 750 "$COSMOS_BASE"
diff --git a/global/pre-tasks.d/014set-cosmos-permissions b/global/pre-tasks.d/014set-cosmos-permissions
new file mode 100755
index 00000000..08992b5f
--- /dev/null
+++ b/global/pre-tasks.d/014set-cosmos-permissions
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# Set Cosmos directory permissions so that
+# the files cannot be read by anyone but root,
+# since it's possible that the directory
+# can contain files that after applying the
+# overlay to / only should be read or writable
+# by root.
+
+set -e
+self=$(basename "$0")
+
+if ! test -d "$COSMOS_BASE"; then
+    test -z "$COSMOS_VERBOSE" || echo "$self: COSMOS_BASE was not found. Aborting change of permissions."
+    exit 0
+fi
+
+args=""
+if [ "x$COSMOS_VERBOSE" = "xy" ]; then
+    args="-v"
+fi
+
+chown ${args} root:root "$COSMOS_BASE"
+chmod ${args} 750 "$COSMOS_BASE"

From 4b93d9c426b3379f9edf63fb25a41a6d233dd20a Mon Sep 17 00:00:00 2001
From: Patrik Lundin <patlu@sunet.se>
Date: Wed, 24 Jan 2024 14:58:21 +0100
Subject: [PATCH 04/15] run-cosmos: support fleetlock unlocking at boot

This extends run-cosmos with a new argument that calls the unlock
function already included in the script as well as using the already
existing lock() function to make sure there is no race between the
bootup process and cron starting a normal run-cosmos process at the same
time.

The oexit() function is added to support exiting with a OK exit value
the same way eexit() is used to signal something is wrong.

This change also adds the systemd unit file that runs run-cosmos with the
new fleetlock-unlock argument at boot if fleetlock is configured.

While here fix indentation that was mixed between 3 and 4 spaces: it is
now 4 spaces everywhere.
---
 .../run-cosmos-fleetlock-unlocker.service     |  11 ++
 global/overlay/usr/local/bin/run-cosmos       | 136 ++++++++++++------
 2 files changed, 104 insertions(+), 43 deletions(-)
 create mode 100644 global/overlay/etc/systemd/system/run-cosmos-fleetlock-unlocker.service

diff --git a/global/overlay/etc/systemd/system/run-cosmos-fleetlock-unlocker.service b/global/overlay/etc/systemd/system/run-cosmos-fleetlock-unlocker.service
new file mode 100644
index 00000000..7507ff79
--- /dev/null
+++ b/global/overlay/etc/systemd/system/run-cosmos-fleetlock-unlocker.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=run-cosmos fleetlock unlocker
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/run-cosmos fleetlock-unlock
+
+[Install]
+WantedBy=multi-user.target
diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos
index 7da725e6..21bf5b0b 100755
--- a/global/overlay/usr/local/bin/run-cosmos
+++ b/global/overlay/usr/local/bin/run-cosmos
@@ -9,6 +9,7 @@ readonly LOCK_FD=200
 readonly FLEETLOCK_CONFIG=/etc/run-cosmos-fleetlock-conf
 readonly FLEETLOCK_DISABLE_FILE=/etc/run-cosmos-fleetlock-disable
 readonly FLEETLOCK_TOOL=/usr/local/bin/sunet-fleetlock
+readonly FLEETLOCK_UNLOCK_SERVICE=run-cosmos-fleetlock-unlocker.service
 readonly HEALTHCHECK_TOOL=/usr/local/bin/sunet-machine-healthy
 readonly HEALTHCHECK_DISABLE_FILE=/etc/run-cosmos-healthcheck-disable
 
@@ -33,60 +34,109 @@ eexit() {
     exit 1
 }
 
+oexit() {
+    local info_str="$*"
+
+    echo "$info_str"
+    exit 0
+}
+
+fleetlock_enable_unlock_service() {
+    # In case e.g. the unit file has been removed "FragmentPath" will still
+    # return the old filename until daemon-reload is called, so do that here
+    # before we try checking for the FragmentPath.
+    need_reload=$(systemctl show --property NeedDaemonReload $FLEETLOCK_UNLOCK_SERVICE | awk -F= '{print $2}')
+    if [ "$need_reload" = "yes" ]; then
+        systemctl daemon-reload
+    fi
+
+    unit_file=$(systemctl show --property FragmentPath $FLEETLOCK_UNLOCK_SERVICE | awk -F= '{print $2}')
+    if [ -z "$unit_file" ]; then
+        # No unit file matching the service name, do nothing
+        return 0
+    fi
+
+    # Enable the service if needed
+    systemctl is-enabled --quiet $FLEETLOCK_UNLOCK_SERVICE || systemctl enable --quiet $FLEETLOCK_UNLOCK_SERVICE
+}
+
 fleetlock_lock() {
-   if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then
-      local fleetlock_group=""
-      # shellcheck source=/dev/null
-      . $FLEETLOCK_CONFIG || return 1
-      if [ -z "$fleetlock_group" ]; then
-          echo "Unable to set fleetlock_group"
-	  return 1
-      fi
-      echo "Getting fleetlock lock"
-      $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --lock || return 1
-   fi
-   return 0
+    if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then
+        # Make sure the unlock service is enabled before we take a lock if
+        # cosmos ends up rebooting the machine before fleetlock_unlock() is
+        # called.
+        fleetlock_enable_unlock_service || return 1
+        local fleetlock_group=""
+        # shellcheck source=/dev/null
+        . $FLEETLOCK_CONFIG || return 1
+        if [ -z "$fleetlock_group" ]; then
+            echo "Unable to set fleetlock_group"
+            return 1
+        fi
+        echo "Getting fleetlock lock"
+        $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --lock || return 1
+    fi
+    return 0
 }
 
 fleetlock_unlock() {
-   if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then
-      local fleetlock_group=""
-      # shellcheck source=/dev/null
-      . $FLEETLOCK_CONFIG || return 1
-      if [ -z "$fleetlock_group" ]; then
-          echo "Unable to set fleetlock_group"
-	  return 1
-      fi
-      machine_is_healthy || return 1
-      echo "Releasing fleetlock lock"
-      $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --unlock || return 1
-   fi
-   return 0
+    if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then
+        local fleetlock_group=""
+        # shellcheck source=/dev/null
+        . $FLEETLOCK_CONFIG || return 1
+        if [ -z "$fleetlock_group" ]; then
+            echo "Unable to set fleetlock_group"
+            return 1
+        fi
+        machine_is_healthy || return 1
+        echo "Releasing fleetlock lock"
+        $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --unlock || return 1
+    fi
+    return 0
 }
 
 machine_is_healthy() {
-   if [ ! -f $HEALTHCHECK_DISABLE_FILE ] && [ -x $HEALTHCHECK_TOOL ]; then
-       echo "Running any health checks"
-       $HEALTHCHECK_TOOL || return 1
-   fi
-   return 0
+    if [ ! -f $HEALTHCHECK_DISABLE_FILE ] && [ -x $HEALTHCHECK_TOOL ]; then
+        local fleetlock_healthcheck_timeout=""
+        local optional_args=()
+        # shellcheck source=/dev/null
+        . $FLEETLOCK_CONFIG || return 1
+        if [ -n "$fleetlock_healthcheck_timeout" ]; then
+           optional_args+=("--timeout")
+           optional_args+=("$fleetlock_healthcheck_timeout")
+        fi
+        echo "Running any health checks"
+        $HEALTHCHECK_TOOL "${optional_args[@]}" || return 1
+    fi
+    return 0
 }
 
 main () {
-   lock "$PROGNAME" || eexit "Only one instance of $PROGNAME can run at one time."
-   fleetlock_lock || eexit "Unable to acquire fleetlock lock."
-   cosmos "$@" update
-   cosmos "$@" apply
-   fleetlock_unlock || eexit "Unable to release fleetlock lock."
+    lock "$PROGNAME" || eexit "Only one instance of $PROGNAME can run at one time."
+    fleetlock_lock || eexit "Unable to acquire fleetlock lock."
+    cosmos "$@" update
+    cosmos "$@" apply
+    fleetlock_unlock || eexit "Unable to release fleetlock lock."
 
-   touch /var/run/last-cosmos-ok.stamp
+    touch /var/run/last-cosmos-ok.stamp
 
-   find /var/lib/puppet/reports/ -type f -mtime +10 -print0 | xargs -0 rm -f
+    find /var/lib/puppet/reports/ -type f -mtime +10 -print0 | xargs -0 rm -f
+
+    if [ -f /cosmos-reboot ]; then
+        rm -f /cosmos-reboot
+        reboot
+    fi
 }
 
-main "$@"
-
-if [ -f /cosmos-reboot ]; then
-   rm -f /cosmos-reboot
-   reboot
-fi
+# Most of the time we just pass on any arguments to the underlying cosmos
+# tools, if adding special cases here make sure to not shadow any arguments
+# (like "-v") which users expect to be passed on to cosmos.
+case "$1" in
+    "fleetlock-unlock")
+        lock "$PROGNAME" || oexit "$PROGNAME appears locked by a running run-cosmos, let it handle unlocking instead."
+        fleetlock_unlock || eexit "Unable to release fleetlock lock."
+        ;;
+    *)
+        main "$@"
+        ;;
+esac

From df5558befbf7db9a10bbd7d32d9bd0bea0c57124 Mon Sep 17 00:00:00 2001
From: Patrik Lundin <patlu@sunet.se>
Date: Wed, 24 Jan 2024 15:09:38 +0100
Subject: [PATCH 05/15] Fix another indentation mismatch

---
 global/overlay/usr/local/bin/run-cosmos | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos
index 21bf5b0b..7274b623 100755
--- a/global/overlay/usr/local/bin/run-cosmos
+++ b/global/overlay/usr/local/bin/run-cosmos
@@ -102,8 +102,8 @@ machine_is_healthy() {
         # shellcheck source=/dev/null
         . $FLEETLOCK_CONFIG || return 1
         if [ -n "$fleetlock_healthcheck_timeout" ]; then
-           optional_args+=("--timeout")
-           optional_args+=("$fleetlock_healthcheck_timeout")
+            optional_args+=("--timeout")
+            optional_args+=("$fleetlock_healthcheck_timeout")
         fi
         echo "Running any health checks"
         $HEALTHCHECK_TOOL "${optional_args[@]}" || return 1

From c72f5ccd861c9b38c45eaa1ccb010117b7c1b366 Mon Sep 17 00:00:00 2001
From: Johan Wassberg <jocar@sunet.se>
Date: Fri, 12 Apr 2024 15:32:40 +0200
Subject: [PATCH 06/15] Allow for hosts without class(s)

---
 global/overlay/etc/puppet/cosmos_enc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/global/overlay/etc/puppet/cosmos_enc.py b/global/overlay/etc/puppet/cosmos_enc.py
index dca12d33..7769a9aa 100755
--- a/global/overlay/etc/puppet/cosmos_enc.py
+++ b/global/overlay/etc/puppet/cosmos_enc.py
@@ -26,7 +26,8 @@ found = False
 classes = dict()
 for reg, cls in rules.items():
     if re.search(reg, node_name):
-        classes.update(cls)
+        if cls:
+            classes.update(cls)
         found = True
 
 if not found:

From 4b8b8887f62761759486940b81ea1142af6ae8bb Mon Sep 17 00:00:00 2001
From: Patrik Lundin <patlu@sunet.se>
Date: Mon, 17 Jun 2024 11:54:28 +0200
Subject: [PATCH 07/15] sunet-fleetlock: handle connection errors

In order to handle upgrades of the fleetlock server when running only
one server we need to handle connection errors like connection refused
or timed out errors gracefully.

Because there are several different ways the connection can fail and it
is hard to keep track of them all, just catch everything. We then also
need special handling of our own timeout execption so we are not
accidentally stuck retrying forever.

Also fix so we actually use the request_timeout arg for individual HTTP
requests instead of the global timeout.

While here run isort to keep imports tidy.
---
 global/overlay/usr/local/bin/sunet-fleetlock | 37 +++++++++++++-------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/global/overlay/usr/local/bin/sunet-fleetlock b/global/overlay/usr/local/bin/sunet-fleetlock
index e2ee6d98..5c4e8877 100755
--- a/global/overlay/usr/local/bin/sunet-fleetlock
+++ b/global/overlay/usr/local/bin/sunet-fleetlock
@@ -24,20 +24,21 @@
 # When modifying this code please make sure it is passed through the following
 # tools:
 # ===
+# isort
 # black
 # pylint
 # mypy --strict
 # ===
 
-import platform
-import sys
-import signal
-import time
 import argparse
 import configparser
 import os.path
-from typing import Optional, Union
+import platform
+import signal
+import sys
+import time
 from types import FrameType
+from typing import Optional, Union
 
 import requests
 
@@ -80,19 +81,29 @@ def do_fleetlock_request(
     request_id_key = "request-id"
     request_id = None
 
+    retry_sleep_delay = 1
+
     # Loop forever: we depend on the SIGALRM timout to raise an error if it
     # takes too long
     while True:
         if args.verbose:
             print(f"{operation} POST at url {url}")
 
-        resp = requests.post(
-            url,
-            headers=fleetlock_headers,
-            json=fleetlock_data,
-            timeout=args.timeout,
-            auth=("", config[args.lock_group]["password"]),
-        )
+        try:
+            resp = requests.post(
+                url,
+                headers=fleetlock_headers,
+                json=fleetlock_data,
+                timeout=args.request_timeout,
+                auth=("", config[args.lock_group]["password"]),
+            )
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            if isinstance(e, TimeoutException):
+                # This means our global timer is up, no more time to retry
+                raise e
+            print(f"POST request failed: {e}")
+            time.sleep(retry_sleep_delay)
+            continue
 
         if request_id_key in resp.headers:
             request_id = resp.headers[request_id_key]
@@ -126,7 +137,7 @@ def do_fleetlock_request(
             + f"({request_id_key}: {request_id})"
         )
 
-        time.sleep(1)
+        time.sleep(retry_sleep_delay)
 
 
 def read_config(args: argparse.Namespace) -> Union[configparser.ConfigParser, None]:

From e315282bc55025c199483fbb5c94d7a053d047f0 Mon Sep 17 00:00:00 2001
From: Patrik Lundin <patlu@sunet.se>
Date: Mon, 17 Jun 2024 12:40:12 +0200
Subject: [PATCH 08/15] Use more strict exception checking

This is probably wide enough and we do not need weird extra handling of
our own execption etc.

Thanks to @mickenordin for keeping me honest :).
---
 global/overlay/usr/local/bin/sunet-fleetlock | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/global/overlay/usr/local/bin/sunet-fleetlock b/global/overlay/usr/local/bin/sunet-fleetlock
index 5c4e8877..6f0b5578 100755
--- a/global/overlay/usr/local/bin/sunet-fleetlock
+++ b/global/overlay/usr/local/bin/sunet-fleetlock
@@ -97,10 +97,7 @@ def do_fleetlock_request(
                 timeout=args.request_timeout,
                 auth=("", config[args.lock_group]["password"]),
             )
-        except Exception as e:  # pylint: disable=broad-exception-caught
-            if isinstance(e, TimeoutException):
-                # This means our global timer is up, no more time to retry
-                raise e
+        except requests.exceptions.ConnectionError as e:
             print(f"POST request failed: {e}")
             time.sleep(retry_sleep_delay)
             continue

From bc9d1dc960c321e130ef94e2521cef07bde0a829 Mon Sep 17 00:00:00 2001
From: Patrik Holmqvist <pahol@sunet.se>
Date: Wed, 19 Jun 2024 14:02:24 +0200
Subject: [PATCH 09/15] Use upstream puppet modules for ubuntu24+.

This is how we do it in modern debian so it
makes sense to do it on modern ubuntu as well.
---
 global/pre-tasks.d/030puppet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/global/pre-tasks.d/030puppet b/global/pre-tasks.d/030puppet
index c431f0a3..e6e53649 100755
--- a/global/pre-tasks.d/030puppet
+++ b/global/pre-tasks.d/030puppet
@@ -13,7 +13,7 @@ if ! test -f "${stamp}" -a -f /usr/bin/puppet; then
     . /etc/os-release
 
     # Note: in posix shell, string comparison is done with a single =
-    if [ "${ID}" = "debian" ] && [ "${VERSION_ID}" -ge 12 ]; then
+    if [ "${ID}" = "debian" ] && [ "${VERSION_ID}" -ge 12 ] || ([ "${ID}" = "ubuntu" ] && $(dpkg --compare-versions ${VERSION_ID} ge 24.04)) ; then
       apt-get -y install \
           cron \
           puppet-module-camptocamp-augeas \

From 4231b4ac1df001df835c043eb1ec1896eabc3cdf Mon Sep 17 00:00:00 2001
From: Patrik Holmqvist <pahol@sunet.se>
Date: Wed, 19 Jun 2024 14:07:13 +0200
Subject: [PATCH 10/15] Migrate from legacy fact

This did not work on modern puppet in ubuntu24:
Warning: Interpolation failed with '::lsbdistcodename', but compilation continuing;
New syntax inspiration from:
https://www.puppet.com/docs/puppet/8/hiera_config_yaml_5#configuring_hiera
---
 global/overlay/etc/puppet/hiera.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/global/overlay/etc/puppet/hiera.yaml b/global/overlay/etc/puppet/hiera.yaml
index 3de986b9..a301376f 100644
--- a/global/overlay/etc/puppet/hiera.yaml
+++ b/global/overlay/etc/puppet/hiera.yaml
@@ -21,7 +21,7 @@ hierarchy:
       pkcs7_public_key:  /etc/hiera/eyaml/public_certkey.pkcs7.pem
 
   - name: "Overrides per distribution"
-    path: "dist_%{::lsbdistcodename}_override.yaml"
+    path: "dist_%{facts.os.distro.codename}_override.yaml"
 
   - name: "Data common to whole environment"
     path: "common.yaml"
\ No newline at end of file

From 01768129f0ce4a7551cb39e46e64c034fcd5606d Mon Sep 17 00:00:00 2001
From: Patrik Lundin <patlu@sunet.se>
Date: Wed, 3 Jul 2024 13:27:52 +0200
Subject: [PATCH 11/15] fleetlock: configurable lock/unlock timeout

While we already support setting a healthcheck timeout it probably
makes sense to be able to control how long we wait for a
fleetlock_lock() or fleetlock_unlock() call. This becomes important if
only running cosmos once a night or something like that. In that case we
you probably want to give a physical machine more than than 1 minute to
complete a reboot etc.

This can now be controlled by setting fleetlock_lock_timeout and
fleetlock_unlock_timeout in /etc/run-cosmos-fleetlock-conf. Keep in mind
that while it can make sense to increase the time for taking a lock,
releasing a lock should always be fast (either you have it and release
it, or you dont have it and it is a no-op) so setting a long unlock
timeout should probably never be done.

Since we also potentially wait the unlock timeout at boot (if the
fleetlock server is broken etc) that is another reason to keep it
short. The default 1m is probably OK for most uses.
---
 global/overlay/usr/local/bin/run-cosmos | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos
index 7274b623..268fd6a3 100755
--- a/global/overlay/usr/local/bin/run-cosmos
+++ b/global/overlay/usr/local/bin/run-cosmos
@@ -67,14 +67,19 @@ fleetlock_lock() {
         # called.
         fleetlock_enable_unlock_service || return 1
         local fleetlock_group=""
+        local optional_args=()
         # shellcheck source=/dev/null
         . $FLEETLOCK_CONFIG || return 1
         if [ -z "$fleetlock_group" ]; then
             echo "Unable to set fleetlock_group"
             return 1
         fi
+        if [ -n "$fleetlock_lock_timeout" ]; then
+            optional_args+=("--timeout")
+            optional_args+=("$fleetlock_lock_timeout")
+        fi
         echo "Getting fleetlock lock"
-        $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --lock || return 1
+        $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --lock "${optional_args[@]}" || return 1
     fi
     return 0
 }
@@ -82,15 +87,20 @@ fleetlock_lock() {
 fleetlock_unlock() {
     if [ ! -f $FLEETLOCK_DISABLE_FILE ] && [ -f $FLEETLOCK_CONFIG ] && [ -x $FLEETLOCK_TOOL ]; then
         local fleetlock_group=""
+        local optional_args=()
         # shellcheck source=/dev/null
         . $FLEETLOCK_CONFIG || return 1
         if [ -z "$fleetlock_group" ]; then
             echo "Unable to set fleetlock_group"
             return 1
         fi
+        if [ -n "$fleetlock_unlock_timeout" ]; then
+            optional_args+=("--timeout")
+            optional_args+=("$fleetlock_unlock_timeout")
+        fi
         machine_is_healthy || return 1
         echo "Releasing fleetlock lock"
-        $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --unlock || return 1
+        $FLEETLOCK_TOOL --lock-group "$fleetlock_group" --unlock "${optional_args[@]}" || return 1
     fi
     return 0
 }

From aa88795ee09e1fb0ffd1b0ffee948e1ad4f33dd9 Mon Sep 17 00:00:00 2001
From: Patrik Lundin <patlu@sunet.se>
Date: Wed, 3 Jul 2024 14:13:22 +0200
Subject: [PATCH 12/15] sunet-fleetlock: also handle ReadTimeout

Turns out this was not caught by ConnectionError.
---
 global/overlay/usr/local/bin/sunet-fleetlock | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/global/overlay/usr/local/bin/sunet-fleetlock b/global/overlay/usr/local/bin/sunet-fleetlock
index 6f0b5578..64effec0 100755
--- a/global/overlay/usr/local/bin/sunet-fleetlock
+++ b/global/overlay/usr/local/bin/sunet-fleetlock
@@ -97,7 +97,10 @@ def do_fleetlock_request(
                 timeout=args.request_timeout,
                 auth=("", config[args.lock_group]["password"]),
             )
-        except requests.exceptions.ConnectionError as e:
+        except (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.ReadTimeout,
+        ) as e:
             print(f"POST request failed: {e}")
             time.sleep(retry_sleep_delay)
             continue

From fac9a556babf748007db6773d0f2bfc85439b6c2 Mon Sep 17 00:00:00 2001
From: Patrik Holmqvist <pahol@sunet.se>
Date: Mon, 9 Sep 2024 16:52:38 +0200
Subject: [PATCH 13/15] Patch for broken eyaml in ubuntu24.04.

---
 edit-secrets | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/edit-secrets b/edit-secrets
index a2c67ac3..144afc42 100755
--- a/edit-secrets
+++ b/edit-secrets
@@ -73,6 +73,41 @@ EOF
     fi
 }
 
+function patch_broken_eyaml_noble {
+    #
+    # Ubuntu 24.04 (noble) has a hiera-eyaml version that is incompatible with ruby 3.2+ (default in ubuntu24).
+    # This is fixed in hiera-eyaml version 3.3.0: https://github.com/voxpupuli/hiera-eyaml/pull/340/files
+    # https://github.com/voxpupuli/hiera-eyaml/blob/master/CHANGELOG.md
+    # But there is no modern version of hiera-eyanl packaged in debian or ubuntu.
+    # https://github.com/puppetlabs/puppet/wiki/Puppet-8-Compatibility#filedirexists-removed
+    #
+
+    . /etc/os-release
+    if [ "${VERSION_CODENAME}" == "noble" ]; then
+        plugins_file="/usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb"
+        if [ -f $plugins_file ]; then
+            # We only want to try patching the file if it is the known broken version
+            bad_sum="59c6eb910ab2eb44f8c75aeaa79bff097038feb673b5c6bdccde23d9b2a393e2"
+            sum=$(sha256sum $plugins_file | awk '{print $1}')
+            if [ "$sum" == "$bad_sum" ]; then
+                patch --fuzz=0 --directory=/ --strip=0 <<'EOF'
+--- /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb.orig	2022-06-11 16:30:10.000000000 +0000
++++ /usr/share/rubygems-integration/all/gems/hiera-eyaml-3.3.0/lib/hiera/backend/eyaml/subcommands/edit.rb	2024-09-09 14:13:19.306342025 +0000
+@@ -59,7 +59,7 @@
+             Optimist::die "You must specify an eyaml file" if ARGV.empty?
+             options[:source] = :eyaml
+             options[:eyaml] = ARGV.shift
+-            if File.exists? options[:eyaml]
++            if File.exist? options[:eyaml]
+               begin
+                 options[:input_data] = File.read options[:eyaml]
+               rescue
+EOF
+            fi
+        fi
+    fi
+}
+
 function edit_copy_and_commit()
 {
     #
@@ -234,6 +269,7 @@ function edit_eyaml_file()
     done
 
     patch_broken_eyaml
+    patch_broken_eyaml_noble
 
     # save source file for comparision afterwards
     cp "${EYAMLFILE}" "${TMPFILE}"

From 7941e3f9702ef1a18115e4d5d1507a0dd43c5141 Mon Sep 17 00:00:00 2001
From: Patrik Holmqvist <pahol@sunet.se>
Date: Mon, 9 Sep 2024 17:29:31 +0200
Subject: [PATCH 14/15] Merge the 2 patch functions to 1.

---
 edit-secrets | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/edit-secrets b/edit-secrets
index 144afc42..68d3630a 100755
--- a/edit-secrets
+++ b/edit-secrets
@@ -71,14 +71,12 @@ EOF
             fi
         fi
     fi
-}
 
-function patch_broken_eyaml_noble {
     #
     # Ubuntu 24.04 (noble) has a hiera-eyaml version that is incompatible with ruby 3.2+ (default in ubuntu24).
     # This is fixed in hiera-eyaml version 3.3.0: https://github.com/voxpupuli/hiera-eyaml/pull/340/files
     # https://github.com/voxpupuli/hiera-eyaml/blob/master/CHANGELOG.md
-    # But there is no modern version of hiera-eyanl packaged in debian or ubuntu.
+    # But there is no modern version of hiera-eyaml packaged in debian or ubuntu.
     # https://github.com/puppetlabs/puppet/wiki/Puppet-8-Compatibility#filedirexists-removed
     #
 
@@ -269,7 +267,6 @@ function edit_eyaml_file()
     done
 
     patch_broken_eyaml
-    patch_broken_eyaml_noble
 
     # save source file for comparision afterwards
     cp "${EYAMLFILE}" "${TMPFILE}"

From f537508bee0fe76c801805c02cd690ed6944ce8b Mon Sep 17 00:00:00 2001
From: Patrik Lundin <patlu@sunet.se>
Date: Thu, 17 Oct 2024 16:36:45 +0200
Subject: [PATCH 15/15] Do not install ntp with cosmos script

This is handled with sunet::server
---
 global/pre-tasks.d/020common-tools | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/global/pre-tasks.d/020common-tools b/global/pre-tasks.d/020common-tools
index eaca6ea8..aece6464 100755
--- a/global/pre-tasks.d/020common-tools
+++ b/global/pre-tasks.d/020common-tools
@@ -8,7 +8,7 @@ set -e
 stamp="$COSMOS_BASE/stamps/common-tools-v01.stamp"
 
 if ! test -f $stamp; then
-    apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades ntp
+    apt-get -y install vim traceroute tcpdump molly-guard less rsync git-core unattended-upgrades
     update-alternatives --set editor /usr/bin/vim.basic
 
     mkdir -p `dirname $stamp`