From bc17ee13541337bf30fc3dcd2767db73dd131806 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 24 Jan 2023 10:01:59 +0100 Subject: [PATCH 01/26] Don't confuse containers to connect to them self When the hostname pointed to loopback the containers tried to connect to them self instead of the host. --- .../etc/cosmos/apt/bootstrap-cosmos.sh | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index c31cea7..5e27f3d 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -30,7 +30,7 @@ export DEBIAN_FRONTEND='noninteractive' apt-get -y update apt-get -y upgrade -for pkg in rsync git git-core wget gpg; do +for pkg in rsync git git-core wget gpg jq; do # script is running with "set -e", use "|| true" to allow packages to not # exist without stopping the script apt-get -y install $pkg || true @@ -56,16 +56,43 @@ mv -f /etc/rc.local.new /etc/rc.local touch /etc/run-cosmos-at-boot # If this cloud-config is set, it will interfere with our changes to /etc/hosts -if [ -f /etc/cloud/cloud.cfg ]; then - sed -i 's/manage_etc_hosts: true/manage_etc_hosts: false/g' /etc/cloud/cloud.cfg -fi +# The configuration seems to move around between cloud-config versions +for file in /etc/cloud/cloud.cfg /etc/cloud/cloud.cfg.d/01_debian_cloud.cfg; do + if [ -f ${file} ]; then + sed -i 's/manage_etc_hosts: true/manage_etc_hosts: false/g' ${file} + fi +done -# Remove potential $hostname.novalocal line from /etc/hosts, added by cloud-init -sed -i.bak -e "s/^127\.0\.1\.1 $(hostname)\..*novalocal.*//1" /etc/hosts +# Remove potential $hostname.novalocal, added by cloud-init or Debian default +# from /etc/hosts. We add our own further down. +# +# From # https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution: +# "For a system with a permanent IP address, that permanent IP address should +# be used here instead of 127.0.1.1." +sed -i.bak -e "/127\.0\.1\.1/d" /etc/hosts + +vendor=$(lsb_release -is) +version=$(lsb_release -rs) +min_version=1337 +host_ip=127.0.1.1 +if [ "${vendor}" = "Ubuntu" ]; then + min_version=20.04 +elif [ "${vendor}" = "Debian" ]; then + min_version=11 +fi hostname $cmd_hostname short=`echo ${cmd_hostname} | awk -F. '{print $1}'` -echo "127.0.1.1 ${cmd_hostname} ${short}" >> /etc/hosts +# Only change behavior on modern OS where `ip -j` outputs a json predictuble +# enought to work with. +# +# Use `dpkg` to easier compare ubuntu versions. +if dpkg --compare-versions "${version}" "ge" "${min_version}"; then + # When hostname pointed to loopback in /etc/hosts containers running on the + # host tried to connect to the container itself instead of the host. + host_ip=$(ip -j address show "$(ip -j route show default | jq -r '.[0].dev')" | jq -r .[0].addr_info[0].local) +fi +echo "${host_ip} ${cmd_hostname} ${short}" >> /etc/hosts # Set up cosmos models. They are in the order of most significant first, so we want # From 57b302a299f14caead2ddeba285093aeab3cf760 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Mon, 30 Jan 2023 11:21:01 +0100 Subject: [PATCH 02/26] Only update the database when needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this fix: ``` ➜ nunoc-ops git:(master) make db On branch master Your branch is up to date with 'origin/master'. nothing to commit, working tree clean make: *** [db] Error 1 ``` With this fix: ``` ➜ swamid-ops git:(master) ✗ make db make: Nothing to be done for `db'. ``` Will make it easier to only use `make tag` when to sign changes. --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8936489..7ad799e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,9 @@ cosmos: upgrade: fab upgrade -db: +db: global/overlay/etc/puppet/cosmos-db.yaml + +global/overlay/etc/puppet/cosmos-db.yaml: global/overlay/etc/puppet/cosmos-rules.yaml: @python ./fabfile/db.py > global/overlay/etc/puppet/cosmos-db.yaml @git add global/overlay/etc/puppet/cosmos-db.yaml && git commit -m "update db" global/overlay/etc/puppet/cosmos-db.yaml From d604d2fab58f2c7432c0cc5fd8ec532a2c7f0795 Mon Sep 17 00:00:00 2001 From: Leif Johansson Date: Mon, 9 Jan 2017 22:03:38 +0100 Subject: [PATCH 03/26] set no-protection on the private key --- global/pre-tasks.d/040hiera-gpg | 1 + 1 file changed, 1 insertion(+) diff --git a/global/pre-tasks.d/040hiera-gpg b/global/pre-tasks.d/040hiera-gpg index e5de6da..3aa3037 100755 --- a/global/pre-tasks.d/040hiera-gpg +++ b/global/pre-tasks.d/040hiera-gpg @@ -44,6 +44,7 @@ Name-Comment: Hiera GPG key Name-Email: root@`hostname --fqdn` Expire-Date: 0 # Do a commit here, so that we can later print "done" :-) +%no-protection %commit %echo done EOF From 4601e0bf08ade3b7d94ed436aad9d011a5b62d2d Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Mon, 30 Jan 2023 14:12:13 +0100 Subject: [PATCH 04/26] make sure we get clean checkouts --- global/post-tasks.d/018packages | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index ee1889f..a48efbe 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -42,7 +42,8 @@ if [ -f $CONFIG -o $LOCALCONFIG ]; then if [ "$src" != "$(git config remote.origin.url)" ]; then git config remote.origin.url $src fi - git pull -q + # Update repo and clean out any local inconsistencies + git pull -q || (git fetch && git reset --hard) else continue fi From e212b6f56f73f3038a4fcfd7921528cb8335a25b Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Tue, 31 Jan 2023 08:19:41 +0100 Subject: [PATCH 05/26] Support master branch being renamed to main Fixes: ``` 70run-post-tasks: invoking /var/cache/cosmos/model/post-tasks.d/018packages Your configuration specifies to merge with the ref 'refs/heads/master' from the remote, but no such ref was fetched. ``` --- global/post-tasks.d/018packages | 2 ++ 1 file changed, 2 insertions(+) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index a48efbe..39569b2 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -42,6 +42,8 @@ if [ -f $CONFIG -o $LOCALCONFIG ]; then if [ "$src" != "$(git config remote.origin.url)" ]; then git config remote.origin.url $src fi + # Support master branch being renamed to main + git branch --all | grep -q '^[[:space:]]*remotes/origin/main$' && git checkout main # Update repo and clean out any local inconsistencies git pull -q || (git fetch && git reset --hard) else From 281f818062bc00d81344c8c24026abaf130f024d Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Wed, 1 Feb 2023 11:19:49 +0100 Subject: [PATCH 06/26] Update setup_cosmos_modules.example * Only update cosmos-modules.conf if a change is needed * Do an atomic replace of the file if updating it * Add a note at the top mentioning that it is tool-generated * Make pylint and black happy Tested in ubuntu 18.04. --- docs/setup_cosmos_modules.example | 318 +++++++++++++++++++----------- 1 file changed, 200 insertions(+), 118 deletions(-) diff --git a/docs/setup_cosmos_modules.example b/docs/setup_cosmos_modules.example index 9b4c7e2..6b1b9c6 100755 --- a/docs/setup_cosmos_modules.example +++ b/docs/setup_cosmos_modules.example @@ -1,134 +1,216 @@ #!/usr/bin/env python3 +""" Write out a puppet cosmos-modules.conf """ + +import hashlib +import os +import os.path +import sys try: from configobj import ConfigObj - os_info = ConfigObj("/etc/os-release") + OS_INFO = ConfigObj("/etc/os-release") except (IOError, ModuleNotFoundError): - os_info = None + OS_INFO = None -modulesfile: str = "/etc/puppet/cosmos-modules.conf" -modules: dict = { - "concat": { - "repo": "https://github.com/SUNET/puppetlabs-concat.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "stdlib": { - "repo": "https://github.com/SUNET/puppetlabs-stdlib.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "cosmos": { - "repo": "https://github.com/SUNET/puppet-cosmos.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "ufw": { - "repo": "https://github.com/SUNET/puppet-module-ufw.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "apt": { - "repo": "https://github.com/SUNET/puppetlabs-apt.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "vcsrepo": { - "repo": "https://github.com/SUNET/puppetlabs-vcsrepo.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "xinetd": { - "repo": "https://github.com/SUNET/puppetlabs-xinetd.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "python": { - "repo": "https://github.com/SUNET/puppet-python.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "hiera-gpg": { - "repo": "https://github.com/SUNET/hiera-gpg.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "pound": { - "repo": "https://github.com/SUNET/puppet-pound.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "augeas": { - "repo": "https://github.com/SUNET/puppet-augeas.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "bastion": { - "repo": "https://github.com/SUNET/puppet-bastion.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "pyff": { - "repo": "https://github.com/samlbits/puppet-pyff.git", - "upgrade": "yes", - "tag": "puppet-pyff-*", - }, - "dhcp": { - "repo": "https://github.com/SUNET/puppetlabs-dhcp.git", - "upgrade": "yes", - "tag": "sunet_dev-2*", - }, - "varnish": { - "repo": "https://github.com/samlbits/puppet-varnish.git", - "upgrade": "yes", - "tag": "puppet-varnish-*", - }, - "apparmor": { - "repo": "https://github.com/SUNET/puppet-apparmor.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "docker": { - "repo": "https://github.com/SUNET/garethr-docker.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "network": { - "repo": "https://github.com/SUNET/attachmentgenie-network.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "sunet": { - "repo": "https://github.com/SUNET/puppet-sunet.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "sysctl": { - "repo": "https://github.com/SUNET/puppet-sysctl.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "nagioscfg": { - "repo": "https://github.com/SUNET/puppet-nagioscfg.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, -} +def get_file_hash(modulesfile): + """ + Based on https://github.com/python/cpython/pull/31930: should use + hashlib.file_digest() but it is only available in python 3.11 + """ + try: + with open(modulesfile, "rb") as fileobj: + digestobj = hashlib.sha256() + _bufsize = 2**18 + buf = bytearray(_bufsize) # Reusable buffer to reduce allocations. + view = memoryview(buf) + while True: + size = fileobj.readinto(buf) + if size == 0: + break # EOF + digestobj.update(view[:size]) + except FileNotFoundError: + return "" -# When/if we want we can do stuff to modules here -if os_info: - if os_info["VERSION_CODENAME"] == "bullseye": - pass + return digestobj.hexdigest() -with open(modulesfile, "w") as fh: + +def get_list_hash(file_lines): + """Get hash of list contents""" + + file_lines_hash = hashlib.sha256() + for line in file_lines: + file_lines_hash.update(line) + + return file_lines_hash.hexdigest() + + +def create_file_content(modules): + """ + Write out the expected file contents to a list so we can check the + expected checksum before writing anything + """ + file_lines = [] + file_lines.append( + "# Generated by {}\n".format( # pylint: disable=consider-using-f-string + os.path.basename(sys.argv[0]) + ).encode("utf-8") + ) for key in modules: - fh.write( - "{0:11} {1} {2} {3}\n".format( + file_lines.append( + "{0:11} {1} {2} {3}\n".format( # pylint: disable=consider-using-f-string key, modules[key]["repo"], modules[key]["upgrade"], modules[key]["tag"], - ) + ).encode("utf-8") ) + + return file_lines + + +def main(): + """Starting point of the program""" + + modulesfile: str = "/etc/puppet/cosmos-modules.conf" + modulesfile_tmp: str = modulesfile + ".tmp" + + modules: dict = { + "concat": { + "repo": "https://github.com/SUNET/puppetlabs-concat.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "stdlib": { + "repo": "https://github.com/SUNET/puppetlabs-stdlib.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "cosmos": { + "repo": "https://github.com/SUNET/puppet-cosmos.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "ufw": { + "repo": "https://github.com/SUNET/puppet-module-ufw.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "apt": { + "repo": "https://github.com/SUNET/puppetlabs-apt.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "vcsrepo": { + "repo": "https://github.com/SUNET/puppetlabs-vcsrepo.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "xinetd": { + "repo": "https://github.com/SUNET/puppetlabs-xinetd.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "python": { + "repo": "https://github.com/SUNET/puppet-python.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "hiera-gpg": { + "repo": "https://github.com/SUNET/hiera-gpg.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "pound": { + "repo": "https://github.com/SUNET/puppet-pound.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "augeas": { + "repo": "https://github.com/SUNET/puppet-augeas.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "bastion": { + "repo": "https://github.com/SUNET/puppet-bastion.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "pyff": { + "repo": "https://github.com/samlbits/puppet-pyff.git", + "upgrade": "yes", + "tag": "puppet-pyff-*", + }, + "dhcp": { + "repo": "https://github.com/SUNET/puppetlabs-dhcp.git", + "upgrade": "yes", + "tag": "sunet_dev-2*", + }, + "varnish": { + "repo": "https://github.com/samlbits/puppet-varnish.git", + "upgrade": "yes", + "tag": "puppet-varnish-*", + }, + "apparmor": { + "repo": "https://github.com/SUNET/puppet-apparmor.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "docker": { + "repo": "https://github.com/SUNET/garethr-docker.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "network": { + "repo": "https://github.com/SUNET/attachmentgenie-network.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "sunet": { + "repo": "https://github.com/SUNET/puppet-sunet.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "sysctl": { + "repo": "https://github.com/SUNET/puppet-sysctl.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "nagioscfg": { + "repo": "https://github.com/SUNET/puppet-nagioscfg.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + } + + # When/if we want we can do stuff to modules here + if OS_INFO: + if OS_INFO["VERSION_CODENAME"] == "bullseye": + pass + + # Build list of expected file content + file_lines = create_file_content(modules) + + # Get hash of the list + list_hash = get_list_hash(file_lines) + + # Get hash of the existing file on disk + file_hash = get_file_hash(modulesfile) + + # Update the file if necessary + if list_hash != file_hash: + # Since we are reading the file with 'rb' when computing our hash use 'wb' when + # writing so we dont end up creating a file that does not match the + # expected hash + with open(modulesfile_tmp, "wb") as fileobj: + for line in file_lines: + fileobj.write(line) + + # Rename it in place so the update is atomic for anything else trying to + # read the file + os.rename(modulesfile_tmp, modulesfile) + + +if __name__ == "__main__": + main() From 49ba964897da59f4208a566dfa1a41c9fe417a93 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 2 Feb 2023 11:45:30 +0100 Subject: [PATCH 07/26] Wrap cosmos in scriptherder if available nunoc-ops and others has been doing this for ages by just modifing the cron file. --- global/overlay/etc/cron.d/cosmos | 2 +- global/overlay/usr/local/libexec/cosmos-cron-wrapper | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 global/overlay/usr/local/libexec/cosmos-cron-wrapper diff --git a/global/overlay/etc/cron.d/cosmos b/global/overlay/etc/cron.d/cosmos index 4eab8de..38d14f5 100644 --- a/global/overlay/etc/cron.d/cosmos +++ b/global/overlay/etc/cron.d/cosmos @@ -1,4 +1,4 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -*/15 * * * * root test -f /etc/no-automatic-cosmos || /usr/local/bin/run-cosmos +*/15 * * * * root test -f /etc/no-automatic-cosmos || /usr/local/libexec/cosmos-cron-wrapper diff --git a/global/overlay/usr/local/libexec/cosmos-cron-wrapper b/global/overlay/usr/local/libexec/cosmos-cron-wrapper new file mode 100644 index 0000000..2df85b5 --- /dev/null +++ b/global/overlay/usr/local/libexec/cosmos-cron-wrapper @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +RUN_COSMOS=/usr/local/bin/run-cosmos +SCRIPTHERDER_CMD='' + +if [ -f /usr/local/bin/scriptherder ]; then + SCRIPTHERDER_CMD='/usr/local/bin/scriptherder --mode wrap --syslog --name cosmos --' +fi + +${SCRIPTHERDER_CMD} ${RUN_COSMOS} From 84b29e4eaaaf6dd45e9ef10b38c26962d3899dc4 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 2 Feb 2023 11:49:10 +0100 Subject: [PATCH 08/26] Executable --- global/overlay/usr/local/libexec/cosmos-cron-wrapper | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 global/overlay/usr/local/libexec/cosmos-cron-wrapper diff --git a/global/overlay/usr/local/libexec/cosmos-cron-wrapper b/global/overlay/usr/local/libexec/cosmos-cron-wrapper old mode 100644 new mode 100755 From 17288b9d0be3c7491f3dbec93e3bd0c052af3b59 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Thu, 2 Feb 2023 18:32:18 +0100 Subject: [PATCH 09/26] init --- docs/setup_cosmos_modules.eduid.example | 300 ++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100755 docs/setup_cosmos_modules.eduid.example diff --git a/docs/setup_cosmos_modules.eduid.example b/docs/setup_cosmos_modules.eduid.example new file mode 100755 index 0000000..2b9dfdc --- /dev/null +++ b/docs/setup_cosmos_modules.eduid.example @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +# +# This script is responsible for creating/updating /etc/puppet/cosmos-modules.conf. +# +# If this script exits without creating that file, a default list of modules will be +# selected (by post-tasks.d/010cosmos-modules, the script that invokes this script). +# +# NOTES ABOUT THE IMPLEMENTATION: +# +# - Avoid any third party modules. We want this script to be re-usable in all ops-repos. +# - To make merging easier, try to keep all local alterations in the local_* functions. +# - Format with black and isort. Line width 120. +# - You probably ONLY want to change things in the local_get_modules_hook() function. +# + +import argparse +import csv +import json +import logging +import logging.handlers +import os +import re +import socket +import sys +from pathlib import Path +from typing import Dict, NewType, Optional, cast + +from pkg_resources import parse_version + +logger = logging.getLogger(__name__) # will be overwritten by _setup_logging() + +# Set up types for data that is passed around in functions in this script. +# Need to use Dict (not dict) here since these aren't stripped by strip-hints, and doesn't work on Ubuntu <= 20.04. +Arguments = NewType("Arguments", argparse.Namespace) +OSInfo = Dict[str, str] +HostInfo = Dict[str, Optional[str]] +Modules = Dict[str, Dict[str, str]] + + +def parse_args() -> Arguments: + """ + Parse the command line arguments + """ + parser = argparse.ArgumentParser( + description="Setup cosmos-modules.conf", + add_help=True, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument("--debug", dest="debug", action="store_true", default=False, help="Enable debug operation") + parser.add_argument( + "--filename", dest="filename", type=str, default="/etc/puppet/cosmos-modules.conf", help="Filename to write to" + ) + + return cast(Arguments, parser.parse_args()) + + +def get_os_info() -> OSInfo: + """Load info about the current OS (distro, release etc.)""" + os_info: OSInfo = {} + if Path("/etc/os-release").exists(): + os_info.update({k.lower(): v for k, v in _parse_bash_vars("/etc/os-release").items()}) + res = local_os_info_hook(os_info) + logger.debug(f"OS info:\n{json.dumps(res, sort_keys=True, indent=4)}") + return res + + +def get_host_info() -> HostInfo: + """Load info about the current host (hostname, fqdn, domain name etc.)""" + try: + fqdn = socket.getfqdn() + hostname = socket.gethostname() + except OSError: + host_info = {} + else: + _domainname = fqdn[len(hostname + ".") :] + + host_info: HostInfo = { + "domainname": _domainname, + "fqdn": fqdn, + "hostname": hostname, + } + res = local_host_info_hook(host_info) + logger.debug(f"Host info: {json.dumps(res, sort_keys=True, indent=4)}") + return res + + +def _parse_bash_vars(path: str) -> dict[str, str]: + """ + Parses a bash script and returns a dictionary representing the + variables declared in that script. + + Source: https://dev.to/htv2012/how-to-parse-bash-variables-b4f + + :param path: The path to the bash script + :return: Variables as a dictionary + """ + with open(path) as stream: + contents = stream.read().strip() + + var_declarations = re.findall(r"^[a-zA-Z0-9_]+=.*$", contents, flags=re.MULTILINE) + reader = csv.reader(var_declarations, delimiter="=") + bash_vars = dict(reader) + return bash_vars + + +def get_modules(os_info: OSInfo, host_info: HostInfo) -> Modules: + """Load the list of default modules. + + This is more or less an inventory of all the modules we have. If you don't want + to use all modules in your OPS repo, you can filter them in the local hook. + + If you want to use a different tag for a module on a specific host/os, you can + do that in the local hook as well. + """ + default_modules = """ + # name repo upgrade tag + apparmor https://github.com/SUNET/puppet-apparmor.git yes sunet-2* + apt https://github.com/SUNET/puppetlabs-apt.git yes sunet-2* + augeas https://github.com/SUNET/puppet-augeas.git yes sunet-2* + bastion https://github.com/SUNET/puppet-bastion.git yes sunet-2* + concat https://github.com/SUNET/puppetlabs-concat.git yes sunet-2* + cosmos https://github.com/SUNET/puppet-cosmos.git yes sunet-2* + dhcp https://github.com/SUNET/puppetlabs-dhcp.git yes sunet_dev-2* + docker https://github.com/SUNET/garethr-docker.git yes sunet-2* + hiera-gpg https://github.com/SUNET/hiera-gpg.git yes sunet-2* + munin https://github.com/SUNET/ssm-munin.git yes sunet-2* + nagioscfg https://github.com/SUNET/puppet-nagioscfg.git yes sunet-2* + network https://github.com/SUNET/attachmentgenie-network.git yes sunet-2* + pound https://github.com/SUNET/puppet-pound.git yes sunet-2* + pyff https://github.com/samlbits/puppet-pyff.git yes puppet-pyff-* + python https://github.com/SUNET/puppet-python.git yes sunet-2* + stdlib https://github.com/SUNET/puppetlabs-stdlib.git yes sunet-2* + sunet https://github.com/SUNET/puppet-sunet.git yes sunet-2* + sysctl https://github.com/SUNET/puppet-sysctl.git yes sunet-2* + ufw https://github.com/SUNET/puppet-module-ufw.git yes sunet-2* + varnish https://github.com/samlbits/puppet-varnish.git yes puppet-varnish-* + vcsrepo https://github.com/SUNET/puppetlabs-vcsrepo.git yes sunet-2* + xinetd https://github.com/SUNET/puppetlabs-xinetd.git yes sunet-2* + """ + modules: Modules = {} + for line in default_modules.splitlines(): + try: + if not line.strip() or line.strip().startswith("#"): + continue + _name, _url, _upgrade, _tag = line.split() + modules[_name] = { + "repo": _url, + "upgrade": _upgrade, + "tag": _tag, + } + except ValueError: + logger.error(f"Failed to parse line: {repr(line)}") + raise + + # Remove the UFW module on Ubuntu >= 22.04 (nftables is used there instead) + if os_info.get("name") == "Ubuntu": + ver = os_info.get("version_id") + if ver: + if parse_version(ver) >= parse_version("22.04"): + logger.debug("Removing UFW module for Ubuntu >= 22.04") + del modules["ufw"] + else: + logger.debug("Keeping UFW module for Ubuntu < 22.04") + else: + logger.debug("Unknown Ubuntu module version, keeping UFW module") + + return local_get_modules_hook(os_info, host_info, modules) + + +def local_os_info_hook(os_info: OSInfo) -> OSInfo: + """Local hook to modify os_info in an OPS repo.""" + # Start local changes in this repository + # End local changes + return os_info + + +def local_host_info_hook(host_info: HostInfo) -> HostInfo: + """Local hook to modify host_info in an OPS repo.""" + # Start local changes in this repository + + # Regular expression to tease apart an eduID hostname + hostname_re = re.compile( + r"""^ + (\w+) # function ('idp', 'apps', ...) + - + (\w+) # site ('tug', 'sthb', ...) + - + (\d+) # 1 for staging, 3 for production + """, + re.VERBOSE, + ) + _hostname = host_info.get("hostname") + if _hostname: + m = hostname_re.match(_hostname) + if m: + _function, _site, _num = m.groups() + host_info["function"] = _function + host_info["site"] = _site + if _num == "1": + host_info["environment"] = "staging" + + # End local changes + return host_info + + +def local_get_modules_hook(os_info: OSInfo, host_info: HostInfo, modules: Modules) -> Modules: + """Local hook to modify default set of modules in an OPS repo.""" + # Start local changes in this repository + + _eduid_modules = { + "apparmor", + "apt", + "augeas", + "bastion", + "concat", + "docker", + "munin", + "stdlib", + "sunet", + "ufw", + } + # Only keep the modules eduID actually uses + modules = {k: v for k, v in modules.items() if k in _eduid_modules} + logger.debug(f"Adding modules: {json.dumps(modules, sort_keys=True, indent=4)}") + + # Use eduID tag for puppet-sunet + modules["sunet"]["tag"] = "eduid-stable-2*" + if host_info.get("environment") == "staging": + modules["sunet"]["tag"] = "eduid_dev-2*" + + # use sunet_dev-2* for some modules in staging + for dev_module in ["munin"]: + if host_info.get("environment") == "staging" and dev_module in modules: + modules[dev_module]["tag"] = "sunet_dev-2*" + + # End local changes + return modules + + +def update_cosmos_modules(filename: str, modules: Modules) -> None: + """Create/update the cosmos-modules.conf file. + + First, we check if the file already have the right content. If so, we do nothing. + """ + content = "# This file is automatically generated by the setup_cosmos_modules script.\n# Do not edit it manually.\n" + for k, v in sorted(modules.items()): + content += f"{k:15} {v['repo']:55} {v['upgrade']:5} {v['tag']}\n" + _file = Path(filename) + if _file.exists(): + # Check if the content is already correct, and avoid updating the file if so (so that the timestamp + # of the file at least indicates when the content was last updated) + with _file.open("r") as f: + current = f.read() + if current == content: + logger.debug(f"{filename} is up to date") + return + + # Create/update the file by writing the content to a temporary file and then renaming it + _tmp_file = _file.with_suffix(".tmp") + with _tmp_file.open("w") as f: + f.write(content) + _tmp_file.rename(_file) + logger.debug(f"Updated {filename}") + + +def _setup_logging(my_name: str, args: Arguments): + level = logging.INFO + if args.debug: + level = logging.DEBUG + logging.basicConfig(level=level, stream=sys.stderr, format="{asctime} | {levelname:7} | {message}", style="{") + global logger + logger = logging.getLogger(my_name) + # If stderr is not a TTY, change the log level of the StreamHandler (stream = sys.stderr above) to ERROR + if not sys.stderr.isatty() and not args.debug: + for this_h in logging.getLogger("").handlers: + this_h.setLevel(logging.ERROR) + if args.debug: + logger.setLevel(logging.DEBUG) + + +def main(my_name: str, args: Arguments) -> bool: + _setup_logging(my_name, args) + + os_info = get_os_info() + host_info = get_host_info() + modules = get_modules(os_info, host_info) + + update_cosmos_modules(args.filename, modules) + + return True + + +if __name__ == "__main__": + my_name = os.path.basename(sys.argv[0]) + args = parse_args() + res = main(my_name, args=args) + if res: + sys.exit(0) + sys.exit(1) From 4c877bc6ea86fe80b6e438de01fb479feedc5b97 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 3 Feb 2023 10:58:25 +0100 Subject: [PATCH 10/26] Syntax error --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7ad799e..fac9f2b 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ upgrade: db: global/overlay/etc/puppet/cosmos-db.yaml -global/overlay/etc/puppet/cosmos-db.yaml: global/overlay/etc/puppet/cosmos-rules.yaml: +global/overlay/etc/puppet/cosmos-db.yaml: global/overlay/etc/puppet/cosmos-rules.yaml @python ./fabfile/db.py > global/overlay/etc/puppet/cosmos-db.yaml @git add global/overlay/etc/puppet/cosmos-db.yaml && git commit -m "update db" global/overlay/etc/puppet/cosmos-db.yaml From 948cc803891580047d3cf73e252fe95ce448f0d8 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Fri, 3 Feb 2023 11:24:55 +0100 Subject: [PATCH 11/26] Update cosmos-puppet-ops.mkd --- docs/cosmos-puppet-ops.mkd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cosmos-puppet-ops.mkd b/docs/cosmos-puppet-ops.mkd index 1d121b8..fc7fa9f 100644 --- a/docs/cosmos-puppet-ops.mkd +++ b/docs/cosmos-puppet-ops.mkd @@ -222,7 +222,7 @@ you'll try to push to the multiverse remote! Finally create a branch for the 'multiverse' upstream so you can merge changes to multiverse: ``` -# git checkout -b multiverse --track multiverse/master +# git checkout -b multiverse --track multiverse/main ``` Note that you can maintain your repo on just about any git hosting platform, including From e08346aa3018fcd50a650b48cafd2c4a1e834659 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Fri, 3 Feb 2023 15:39:49 +0100 Subject: [PATCH 12/26] cleanup, use stamp-file, only run on old OS versions --- global/pre-tasks.d/040hiera-gpg | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/global/pre-tasks.d/040hiera-gpg b/global/pre-tasks.d/040hiera-gpg index 3aa3037..bc1da35 100755 --- a/global/pre-tasks.d/040hiera-gpg +++ b/global/pre-tasks.d/040hiera-gpg @@ -9,12 +9,21 @@ set -e GNUPGHOME=/etc/hiera/gpg export GNUPGHOME +vendor=$(lsb_release -is) +version=$(lsb_release -rs) +# If the OS is Ubuntu 18.04 or newer, or Debian 10 or newer, we don't need to do anything (those use eyaml instead) +test "${vendor}" = "Ubuntu" && dpkg --compare-versions "${version}" "ge" "18.04" && exit 0 +test "${vendor}" = "Debian" && dpkg --compare-versions "${version}" "ge" "10" && exit 0 + +stamp="$COSMOS_BASE/stamps/hiera-gpg-v01.stamp" + +test -f "$stamp" && exit 0 + if [ ! -f /usr/lib/ruby/vendor_ruby/gpgme.rb ]; then apt-get update apt-get -y install ruby-gpgme fi - if [ ! -s $GNUPGHOME/secring.gpg ]; then if [ "x$1" != "x--force" ]; then @@ -35,19 +44,21 @@ if [ ! -s $GNUPGHOME/secring.gpg ]; then chmod 700 $GNUPGHOME TMPFILE=$(mktemp /tmp/hiera-gpg.XXXXXX) - cat > $TMPFILE < "$TMPFILE" < Date: Fri, 3 Feb 2023 15:40:15 +0100 Subject: [PATCH 13/26] install eyaml on Ubuntu from 18.04 and Debian from version 10 --- global/pre-tasks.d/040hiera-eyaml | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 global/pre-tasks.d/040hiera-eyaml diff --git a/global/pre-tasks.d/040hiera-eyaml b/global/pre-tasks.d/040hiera-eyaml new file mode 100755 index 0000000..1f2758d --- /dev/null +++ b/global/pre-tasks.d/040hiera-eyaml @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Set up eyaml for Hiera +# + +set -e + +EYAMLDIR=/etc/hiera/eyaml + +vendor=$(lsb_release -is) +version=$(lsb_release -rs) +# eyaml is only used on Ubuntu 20.04 and newer, and Debian 11 and newer (earlier OSes use hiera-gpg instead) +test "${vendor}" = "Ubuntu" && dpkg --compare-versions "${version}" "lt" "18.04" && exit 0 +test "${vendor}" = "Debian" && dpkg --compare-versions "${version}" "lt" "10" && exit 0 + +stamp="$COSMOS_BASE/stamps/hiera-eyaml-v01.stamp" + +test -f "$stamp" && exit 0 + +if [ ! -f /usr/bin/eyaml ] || [ ! -d /usr/share/doc/yaml-mode ]; then + apt-get update + apt-get -y install hiera-eyaml yaml-mode +fi + +if [ ! -f ${EYAMLDIR}/public_certkey.pkcs7.pem ] || [ ! -f ${EYAMLDIR}/private_key.pkcs7.pem ]; then + # hiera-eyaml wants a certificate and public key, not just a public key oddly enough + echo "$0: Generating eyaml key in ${EYAMLDIR} - this might take a while..." + mkdir -p /etc/hiera/eyaml + openssl req -x509 -newkey rsa:4096 -keyout ${EYAMLDIR}/private_key.pkcs7.pem \ + -out ${EYAMLDIR}/public_certkey.pkcs7.pem -days 3653 -nodes -sha256 \ + -subj "/C=SE/O=SUNET/OU=EYAML/CN=$(hostname)" + rm -f ${EYAMLDIR}/public_key.pkcs7.pem # cleanup +fi + +mkdir -p "$(dirname "${stamp}")" +touch "$stamp" From 25463e6013a2d387e48916b496a860cfe7d56ae1 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Fri, 3 Feb 2023 16:04:51 +0100 Subject: [PATCH 14/26] respect COSMOS_VERBOSE --- global/pre-tasks.d/015set-overlay-permissions | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/global/pre-tasks.d/015set-overlay-permissions b/global/pre-tasks.d/015set-overlay-permissions index 373ef68..37f9844 100755 --- a/global/pre-tasks.d/015set-overlay-permissions +++ b/global/pre-tasks.d/015set-overlay-permissions @@ -15,5 +15,9 @@ if ! test -d "$MODEL_OVERLAY"; then fi if [ -d "$MODEL_OVERLAY/root" ]; then - chmod -v 0700 "$MODEL_OVERLAY"/root + args="" + if [ "x$COSMOS_VERBOSE" = "xy" ]; then + args="-v" + fi + chmod ${args} 0700 "$MODEL_OVERLAY"/root fi From 708c6c1b64faa7e175133bc9d96dc2d082708477 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Fri, 3 Feb 2023 16:05:09 +0100 Subject: [PATCH 15/26] add set -e, and do some shellcheck cleanup --- global/post-tasks.d/030puppet | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet index af45005..561ebc4 100755 --- a/global/post-tasks.d/030puppet +++ b/global/post-tasks.d/030puppet @@ -1,13 +1,15 @@ #!/bin/sh +set -e + if [ "x$COSMOS_VERBOSE" = "xy" ]; then args="--verbose --show_diff" else args="--logdest=syslog" fi -if [ -f /usr/bin/puppet -a -d /etc/puppet/manifests ]; then - for m in `find /etc/puppet/manifests -name \*.pp`; do +if [ -f /usr/bin/puppet ] && [ -d /etc/puppet/manifests ]; then + find /etc/puppet/manifests -name \*.pp | while read -r m; do test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" puppet apply $args $m done From 3988f5beb000cb48bc90e775ad5bc6afed60644f Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Mon, 6 Feb 2023 16:41:04 +0100 Subject: [PATCH 16/26] shellcheck fixes --- global/overlay/usr/local/bin/run-cosmos | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos index 5f2cbc1..fdfb85d 100755 --- a/global/overlay/usr/local/bin/run-cosmos +++ b/global/overlay/usr/local/bin/run-cosmos @@ -16,29 +16,29 @@ lock() { eval "exec $fd>$lock_file" # acquier the lock - flock -n $fd \ + flock -n "$fd" \ && return 0 \ || return 1 } eexit() { - local error_str="$@" + local error_str="$*" - echo $error_str + echo "$error_str" exit 1 } main () { - lock $PROGNAME || eexit "Only one instance of $PROGNAME can run at one time." - cosmos $* update - cosmos $* apply + lock "$PROGNAME" || eexit "Only one instance of $PROGNAME can run at one time." + cosmos "$@" update + cosmos "$@" apply touch /var/run/last-cosmos-ok.stamp - find /var/lib/puppet/reports/ -type f -mtime +10 | xargs rm -f + find /var/lib/puppet/reports/ -type f -mtime +10 -print0 | xargs -0 rm -f } -main $* +main "$@" if [ -f /cosmos-reboot ]; then rm -f /cosmos-reboot From 79606f2a6d1919d30902769b58dd61c43b31a023 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Mon, 6 Feb 2023 16:45:53 +0100 Subject: [PATCH 17/26] check for /etc/no-automatic-cosmos in the wrapper, and allow arguments to be passed --- global/overlay/etc/cron.d/cosmos | 2 +- global/overlay/usr/local/libexec/cosmos-cron-wrapper | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/global/overlay/etc/cron.d/cosmos b/global/overlay/etc/cron.d/cosmos index 38d14f5..e7abc17 100644 --- a/global/overlay/etc/cron.d/cosmos +++ b/global/overlay/etc/cron.d/cosmos @@ -1,4 +1,4 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -*/15 * * * * root test -f /etc/no-automatic-cosmos || /usr/local/libexec/cosmos-cron-wrapper +*/15 * * * * root /usr/local/libexec/cosmos-cron-wrapper diff --git a/global/overlay/usr/local/libexec/cosmos-cron-wrapper b/global/overlay/usr/local/libexec/cosmos-cron-wrapper index 2df85b5..ae66810 100755 --- a/global/overlay/usr/local/libexec/cosmos-cron-wrapper +++ b/global/overlay/usr/local/libexec/cosmos-cron-wrapper @@ -1,10 +1,12 @@ #!/usr/bin/env bash -RUN_COSMOS=/usr/local/bin/run-cosmos +test -f /etc/no-automatic-cosmos && exit 0 + +RUN_COSMOS='/usr/local/bin/run-cosmos' SCRIPTHERDER_CMD='' -if [ -f /usr/local/bin/scriptherder ]; then +if [ -x /usr/local/bin/scriptherder ]; then SCRIPTHERDER_CMD='/usr/local/bin/scriptherder --mode wrap --syslog --name cosmos --' fi -${SCRIPTHERDER_CMD} ${RUN_COSMOS} +exec ${SCRIPTHERDER_CMD} ${RUN_COSMOS} "$@" From 12b2412ea78cc5b8302a0be5608d8f10a972f917 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Mon, 6 Feb 2023 17:12:01 +0100 Subject: [PATCH 18/26] run cron at boot too, to e.g. get new firewall rules installed --- global/overlay/etc/cron.d/cosmos | 2 ++ 1 file changed, 2 insertions(+) diff --git a/global/overlay/etc/cron.d/cosmos b/global/overlay/etc/cron.d/cosmos index e7abc17..3840f8c 100644 --- a/global/overlay/etc/cron.d/cosmos +++ b/global/overlay/etc/cron.d/cosmos @@ -2,3 +2,5 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin */15 * * * * root /usr/local/libexec/cosmos-cron-wrapper + +@reboot root sleep 30; /usr/local/libexec/cosmos-cron-wrapper From cae694d1ceea9a94acda4d595bf5fbeae2d609c6 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 7 Feb 2023 08:49:31 +0100 Subject: [PATCH 19/26] Make sure of separator We use the separator later on to determine where the yaml document starts. `eyaml edit` adds the separator to new (non-existing) files by itself but since we want to create the file before in order to diff later the separator needs to be added in order to get a valid document. --- edit-secrets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edit-secrets b/edit-secrets index b4d816c..a2c67ac 100755 --- a/edit-secrets +++ b/edit-secrets @@ -151,7 +151,7 @@ function edit_file_on_host() { edit_gpg_file ${SECRETFILE} elif [ -f /etc/hiera/eyaml/public_certkey.pkcs7.pem ]; then # default to eyaml if the key exists and none of the secrets-file above exist - touch ${EYAMLFILE} + echo "---" > ${EYAMLFILE} edit_eyaml_file ${EYAMLFILE} fi } From c400bba97d8dae9c422937691ab1c174277fe77d Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 14:21:29 +0100 Subject: [PATCH 20/26] remove 'make db' The db-file, essentially providing reverse lookup of classes to host names, is only used by some Nagios configuration instances and causes continuing operational headaches in those ops-repos. It should be kept/refactored to only apply to the monitoring hosts in the cases where it is used, but we don't want any new ops-repos to use it hence it should be removed from upstream multiverse. --- fabfile/db.py | 49 ------------------------- global/overlay/etc/puppet/cosmos_enc.py | 35 ++++++++++++++---- 2 files changed, 27 insertions(+), 57 deletions(-) delete mode 100644 fabfile/db.py diff --git a/fabfile/db.py b/fabfile/db.py deleted file mode 100644 index 67b6645..0000000 --- a/fabfile/db.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import yaml -import re - -def _all_hosts(): - return filter(lambda fn: '.' in fn and not fn.startswith('.') and os.path.isdir(fn),os.listdir(".")) - -def _load_db(): - rules = dict() - rules_file = "cosmos-rules.yaml"; - if os.path.exists(rules_file): - with open(rules_file) as fd: - rules.update(yaml.load(fd)) - - all_hosts = _all_hosts() - - members = dict() - for node_name in all_hosts: - for reg,cls in rules.iteritems(): - if re.match(reg,node_name): - for cls_name in cls.keys(): - h = members.get(cls_name,[]) - h.append(node_name) - members[cls_name] = h - members['all'] = all_hosts - - classes = dict() - for node_name in all_hosts: - node_classes = dict() - for reg,cls in rules.iteritems(): - if re.match(reg,node_name): - node_classes.update(cls) - classes[node_name] = node_classes - - # Sort member lists for a more easy to read diff - for cls in members.keys(): - members[cls].sort() - - return dict(classes=classes,members=members) - -_db = None -def cosmos_db(): - global _db - if _db is None: - _db = _load_db() - return _db - -if __name__ == '__main__': - print yaml.dump(cosmos_db()) diff --git a/global/overlay/etc/puppet/cosmos_enc.py b/global/overlay/etc/puppet/cosmos_enc.py index 852fb25..dca12d3 100755 --- a/global/overlay/etc/puppet/cosmos_enc.py +++ b/global/overlay/etc/puppet/cosmos_enc.py @@ -1,18 +1,37 @@ #!/usr/bin/env python3 +# +# Puppet 'External Node Classifier' to tell puppet what classes to apply to this node. +# +# Docs: https://puppet.com/docs/puppet/5.3/nodes_external.html +# -import sys -import yaml import os import re +import sys + +import yaml + +rules_path = os.environ.get("COSMOS_RULES_PATH", "/etc/puppet") node_name = sys.argv[1] -db_file = os.environ.get("COSMOS_ENC_DB","/etc/puppet/cosmos-db.yaml") -db = dict(classes=dict()) +rules = dict() +for p in rules_path.split(":"): + rules_file = os.path.join(p, "cosmos-rules.yaml") + if os.path.exists(rules_file): + with open(rules_file) as fd: + rules.update(yaml.safe_load(fd)) -if os.path.exists(db_file): - with open(db_file) as fd: - db.update(yaml.load(fd)) +found = False +classes = dict() +for reg, cls in rules.items(): + if re.search(reg, node_name): + classes.update(cls) + found = True -print(yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[]))))) +if not found: + sys.stderr.write(f"{sys.argv[0]}: {node_name} not found in cosmos-rules.yaml\n") +print("---\n" + yaml.dump(dict(classes=classes))) + +sys.exit(0) From 1bddf21049e8ded369e6319de78adb0ad9b880f5 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 15:04:01 +0100 Subject: [PATCH 21/26] remove 'make db' target as well --- Makefile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Makefile b/Makefile index fac9f2b..a284f95 100644 --- a/Makefile +++ b/Makefile @@ -6,13 +6,7 @@ cosmos: upgrade: fab upgrade -db: global/overlay/etc/puppet/cosmos-db.yaml - -global/overlay/etc/puppet/cosmos-db.yaml: global/overlay/etc/puppet/cosmos-rules.yaml - @python ./fabfile/db.py > global/overlay/etc/puppet/cosmos-db.yaml - @git add global/overlay/etc/puppet/cosmos-db.yaml && git commit -m "update db" global/overlay/etc/puppet/cosmos-db.yaml - -tag: db +tag: ./bump-tag test_in_docker: From 252d478e2d309ade7c3bddf63bb3110a56712794 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 16:09:25 +0100 Subject: [PATCH 22/26] cosmos-db.yaml is no more --- host-puppet-conf-test | 1 - 1 file changed, 1 deletion(-) diff --git a/host-puppet-conf-test b/host-puppet-conf-test index e72008c..0844d08 100755 --- a/host-puppet-conf-test +++ b/host-puppet-conf-test @@ -24,7 +24,6 @@ then echo "Copying files to host..." rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@$HOSTNAME:/etc/puppet/cosmos-rules.yaml rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@$HOSTNAME:/etc/puppet/manifests/cosmos-site.pp - rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-db.yaml root@$HOSTNAME:/etc/puppet/cosmos-db.yaml rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@$HOSTNAME:/etc/hiera/data/common.yaml # Test if the user has symlinked puppet-sunet correctly From 496b9f4b31d02d101f7e5cf3bcee603ea199d02e Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 16:09:37 +0100 Subject: [PATCH 23/26] shellcheck fixes --- host-puppet-conf-test | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/host-puppet-conf-test b/host-puppet-conf-test index 0844d08..2ddcbe9 100755 --- a/host-puppet-conf-test +++ b/host-puppet-conf-test @@ -17,14 +17,14 @@ PUPPET_ARGS=${PUPPET_ARGS-"--verbose"} # Check if cosmos or puppet is already running on host echo "Checking if puppet or cosmos is already running..." -ssh root@$HOSTNAME ps aux | egrep -v "grep|edit-secrets|gpg-agent" | egrep -q "cosmos|puppet" +ssh root@"$HOSTNAME" ps aux | grep -Ev "grep|edit-secrets|gpg-agent" | grep -Eq "cosmos|puppet" if [ $? -eq 1 ] then echo "Copying files to host..." - rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@$HOSTNAME:/etc/puppet/cosmos-rules.yaml - rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@$HOSTNAME:/etc/puppet/manifests/cosmos-site.pp - rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@$HOSTNAME:/etc/hiera/data/common.yaml + rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@"$HOSTNAME":/etc/puppet/cosmos-rules.yaml + rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@"$HOSTNAME":/etc/puppet/manifests/cosmos-site.pp + rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@"$HOSTNAME":/etc/hiera/data/common.yaml # Test if the user has symlinked puppet-sunet correctly # by first checking if the link exits and then whether @@ -36,7 +36,7 @@ then fi echo "Running puppet apply..." - ssh root@$HOSTNAME /usr/bin/puppet apply $PUPPET_ARGS /etc/puppet/manifests/cosmos-site.pp + ssh root@"$HOSTNAME" /usr/bin/puppet apply $PUPPET_ARGS /etc/puppet/manifests/cosmos-site.pp else echo "Cosmos or puppet already running. Exiting." exit 1 From 5af80933389b713c2ec76425760602509a5cab64 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 16 Feb 2023 07:44:00 +0100 Subject: [PATCH 24/26] Add support for eyaml in Hiera And at the same time remove support for gpg. The modern version of the configuration (v5) has been tested with 20.04 but might work with older dists. --- global/overlay/etc/puppet/hiera.yaml | 36 ++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/global/overlay/etc/puppet/hiera.yaml b/global/overlay/etc/puppet/hiera.yaml index 3663305..3de986b 100644 --- a/global/overlay/etc/puppet/hiera.yaml +++ b/global/overlay/etc/puppet/hiera.yaml @@ -1,21 +1,27 @@ +# Hiera version 5 configuration +# --- -:backends: - - yaml - - gpg +version: 5 +defaults: + datadir: /etc/hiera/data + data_hash: yaml_data -:logger: console +hierarchy: + - name: "Per-node data" + path: "local.yaml" -:hierarchy: - - "%{env}/%{location}/%{calling_module}" - - "%{env}/%{calling_module}" - - local - - secrets.yaml - - common + - name: "Per-group data" + path: "group.yaml" + - name: "Per-host secrets" + path: "local.eyaml" + lookup_key: eyaml_lookup_key + options: + pkcs7_private_key: /etc/hiera/eyaml/private_key.pkcs7.pem + pkcs7_public_key: /etc/hiera/eyaml/public_certkey.pkcs7.pem -:yaml: - :datadir: /etc/hiera/data + - name: "Overrides per distribution" + path: "dist_%{::lsbdistcodename}_override.yaml" -:gpg: - :datadir: /etc/hiera/data - :key_dir: /etc/hiera/gpg + - name: "Data common to whole environment" + path: "common.yaml" \ No newline at end of file From 0f1c5ec93ff3df4ef8f072478a78b7959da500f5 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 15 Mar 2023 13:08:24 +0100 Subject: [PATCH 25/26] Don't throw away v6 route when IP forwarding In environments where we relay on RAs for IPv6 (e.g Safespring) we need to forcely allow RAs even if IP forwarding is enabled by some service(s). E.g docker: https://github.com/SUNET/puppet-sunet/blob/2dc4de00de1d2404d5dffaf17d18723e2b369cd0/templates/dockerhost/systemd_dropin_nftables_ns.conf.erb#L46 --- iaas-setup.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/iaas-setup.sh b/iaas-setup.sh index 4ce578b..a236b69 100755 --- a/iaas-setup.sh +++ b/iaas-setup.sh @@ -75,6 +75,29 @@ if grep -q '^# en_US.UTF-8 UTF-8$' $locale_gen_file; then locale-gen fi +if [ "$(lsb_release -is)" == "Debian" ]; then + interfaces_file='/etc/network/interfaces.d/50-cloud-init' + + if [ -f "${interfaces_file}" ]; then + interface_string='iface ens3 inet6 dhcp' + accept_ra_string=' accept_ra 2' + + if ! grep -qPz "${interface_string}\n${accept_ra_string}" ${interfaces_file} ; then + + # By default net.ipv6.conf.ens3.accept_ra is set to 1 which + # makes the kernel throw a way the IPv6 route when + # net.ipv6.conf.all.forwarding is set to 1 by our service for + # Docker. + echo "Configuring interfaces to always accept Router Advertisements even with IP Forwarding enabled" + sed -i -r "s/(${interface_string})/\1\n${accept_ra_string}/" ${interfaces_file} + else + echo "WARN: Configuration already applied or no match for \"${interface_string}\" in ${interfaces_file}" + fi + else + echo "WARN: ${interfaces_file} not found. File renamed in this image?" + fi +fi + DEBIAN_FRONTEND="noninteractive" apt-get -y update DEBIAN_FRONTEND="noninteractive" apt-get -o Dpkg::Options::="--force-confnew" --fix-broken --assume-yes dist-upgrade reboot From cf2e6b6518cb18ce1ebb8bfcbfd7945ffada873c Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 4 Apr 2023 15:21:15 +0200 Subject: [PATCH 26/26] File provided by Sunet::Dockerhost --- global/overlay/etc/logrotate.d/docker-containers | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 global/overlay/etc/logrotate.d/docker-containers diff --git a/global/overlay/etc/logrotate.d/docker-containers b/global/overlay/etc/logrotate.d/docker-containers deleted file mode 100644 index e9c90b8..0000000 --- a/global/overlay/etc/logrotate.d/docker-containers +++ /dev/null @@ -1,7 +0,0 @@ -/var/lib/docker/containers/*/*.log { - rotate 7 - daily - compress - delaycompress - copytruncate -}