From 17288b9d0be3c7491f3dbec93e3bd0c052af3b59 Mon Sep 17 00:00:00 2001
From: Fredrik Thulin <fredrik@thulin.net>
Date: Thu, 2 Feb 2023 18:32:18 +0100
Subject: [PATCH] init

---
 docs/setup_cosmos_modules.eduid.example | 300 ++++++++++++++++++++++++
 1 file changed, 300 insertions(+)
 create mode 100755 docs/setup_cosmos_modules.eduid.example

diff --git a/docs/setup_cosmos_modules.eduid.example b/docs/setup_cosmos_modules.eduid.example
new file mode 100755
index 0000000..2b9dfdc
--- /dev/null
+++ b/docs/setup_cosmos_modules.eduid.example
@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+#
+# This script is responsible for creating/updating /etc/puppet/cosmos-modules.conf.
+#
+# If this script exits without creating that file, a default list of modules will be
+# selected (by post-tasks.d/010cosmos-modules, the script that invokes this script).
+#
+# NOTES ABOUT THE IMPLEMENTATION:
+#
+#   - Avoid any third party modules. We want this script to be re-usable in all ops-repos.
+#   - To make merging easier, try to keep all local alterations in the local_* functions.
+#   - Format with black and isort. Line width 120.
+#   - You probably ONLY want to change things in the local_get_modules_hook() function.
+#
+
+import argparse
+import csv
+import json
+import logging
+import logging.handlers
+import os
+import re
+import socket
+import sys
+from pathlib import Path
+from typing import Dict, NewType, Optional, cast
+
+from pkg_resources import parse_version
+
+logger = logging.getLogger(__name__)  # will be overwritten by _setup_logging()
+
+# Set up types for data that is passed around in functions in this script.
+# Need to use Dict (not dict) here since these aren't stripped by strip-hints, and doesn't work on Ubuntu <= 20.04.
+Arguments = NewType("Arguments", argparse.Namespace)
+OSInfo = Dict[str, str]
+HostInfo = Dict[str, Optional[str]]
+Modules = Dict[str, Dict[str, str]]
+
+
+def parse_args() -> Arguments:
+    """
+    Parse the command line arguments
+    """
+    parser = argparse.ArgumentParser(
+        description="Setup cosmos-modules.conf",
+        add_help=True,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    parser.add_argument("--debug", dest="debug", action="store_true", default=False, help="Enable debug operation")
+    parser.add_argument(
+        "--filename", dest="filename", type=str, default="/etc/puppet/cosmos-modules.conf", help="Filename to write to"
+    )
+
+    return cast(Arguments, parser.parse_args())
+
+
+def get_os_info() -> OSInfo:
+    """Load info about the current OS (distro, release etc.)"""
+    os_info: OSInfo = {}
+    if Path("/etc/os-release").exists():
+        os_info.update({k.lower(): v for k, v in _parse_bash_vars("/etc/os-release").items()})
+    res = local_os_info_hook(os_info)
+    logger.debug(f"OS info:\n{json.dumps(res, sort_keys=True, indent=4)}")
+    return res
+
+
+def get_host_info() -> HostInfo:
+    """Load info about the current host (hostname, fqdn, domain name etc.)"""
+    try:
+        fqdn = socket.getfqdn()
+        hostname = socket.gethostname()
+    except OSError:
+        host_info = {}
+    else:
+        _domainname = fqdn[len(hostname + ".") :]
+
+        host_info: HostInfo = {
+            "domainname": _domainname,
+            "fqdn": fqdn,
+            "hostname": hostname,
+        }
+    res = local_host_info_hook(host_info)
+    logger.debug(f"Host info: {json.dumps(res, sort_keys=True, indent=4)}")
+    return res
+
+
+def _parse_bash_vars(path: str) -> dict[str, str]:
+    """
+    Parses a bash script and returns a dictionary representing the
+    variables declared in that script.
+
+    Source: https://dev.to/htv2012/how-to-parse-bash-variables-b4f
+
+    :param path: The path to the bash script
+    :return: Variables as a dictionary
+    """
+    with open(path) as stream:
+        contents = stream.read().strip()
+
+    var_declarations = re.findall(r"^[a-zA-Z0-9_]+=.*$", contents, flags=re.MULTILINE)
+    reader = csv.reader(var_declarations, delimiter="=")
+    bash_vars = dict(reader)
+    return bash_vars
+
+
+def get_modules(os_info: OSInfo, host_info: HostInfo) -> Modules:
+    """Load the list of default modules.
+
+    This is more or less an inventory of all the modules we have. If you don't want
+    to use all modules in your OPS repo, you can filter them in the local hook.
+
+    If you want to use a different tag for a module on a specific host/os, you can
+    do that in the local hook as well.
+    """
+    default_modules = """
+        # name      repo                                                  upgrade  tag
+        apparmor    https://github.com/SUNET/puppet-apparmor.git          yes      sunet-2*
+        apt         https://github.com/SUNET/puppetlabs-apt.git           yes      sunet-2*
+        augeas      https://github.com/SUNET/puppet-augeas.git            yes      sunet-2*
+        bastion     https://github.com/SUNET/puppet-bastion.git           yes      sunet-2*
+        concat      https://github.com/SUNET/puppetlabs-concat.git        yes      sunet-2*
+        cosmos      https://github.com/SUNET/puppet-cosmos.git            yes      sunet-2*
+        dhcp        https://github.com/SUNET/puppetlabs-dhcp.git          yes      sunet_dev-2*
+        docker      https://github.com/SUNET/garethr-docker.git           yes      sunet-2*
+        hiera-gpg   https://github.com/SUNET/hiera-gpg.git                yes      sunet-2*
+        munin       https://github.com/SUNET/ssm-munin.git                yes      sunet-2*
+        nagioscfg   https://github.com/SUNET/puppet-nagioscfg.git         yes      sunet-2*
+        network     https://github.com/SUNET/attachmentgenie-network.git  yes      sunet-2*
+        pound       https://github.com/SUNET/puppet-pound.git             yes      sunet-2*
+        pyff        https://github.com/samlbits/puppet-pyff.git           yes      puppet-pyff-*
+        python      https://github.com/SUNET/puppet-python.git            yes      sunet-2*
+        stdlib      https://github.com/SUNET/puppetlabs-stdlib.git        yes      sunet-2*
+        sunet       https://github.com/SUNET/puppet-sunet.git             yes      sunet-2*
+        sysctl      https://github.com/SUNET/puppet-sysctl.git            yes      sunet-2*
+        ufw         https://github.com/SUNET/puppet-module-ufw.git        yes      sunet-2*
+        varnish     https://github.com/samlbits/puppet-varnish.git        yes      puppet-varnish-*
+        vcsrepo     https://github.com/SUNET/puppetlabs-vcsrepo.git       yes      sunet-2*
+        xinetd      https://github.com/SUNET/puppetlabs-xinetd.git        yes      sunet-2*
+    """
+    modules: Modules = {}
+    for line in default_modules.splitlines():
+        try:
+            if not line.strip() or line.strip().startswith("#"):
+                continue
+            _name, _url, _upgrade, _tag = line.split()
+            modules[_name] = {
+                "repo": _url,
+                "upgrade": _upgrade,
+                "tag": _tag,
+            }
+        except ValueError:
+            logger.error(f"Failed to parse line: {repr(line)}")
+            raise
+
+    # Remove the UFW module on Ubuntu >= 22.04 (nftables is used there instead)
+    if os_info.get("name") == "Ubuntu":
+        ver = os_info.get("version_id")
+        if ver:
+            if parse_version(ver) >= parse_version("22.04"):
+                logger.debug("Removing UFW module for Ubuntu >= 22.04")
+                del modules["ufw"]
+            else:
+                logger.debug("Keeping UFW module for Ubuntu < 22.04")
+        else:
+            logger.debug("Unknown Ubuntu module version, keeping UFW module")
+
+    return local_get_modules_hook(os_info, host_info, modules)
+
+
+def local_os_info_hook(os_info: OSInfo) -> OSInfo:
+    """Local hook to modify os_info in an OPS repo."""
+    # Start local changes in this repository
+    # End local changes
+    return os_info
+
+
+def local_host_info_hook(host_info: HostInfo) -> HostInfo:
+    """Local hook to modify host_info in an OPS repo."""
+    # Start local changes in this repository
+
+    # Regular expression to tease apart an eduID hostname
+    hostname_re = re.compile(
+        r"""^
+        (\w+)   # function ('idp', 'apps', ...)
+        -
+        (\w+)   # site ('tug', 'sthb', ...)
+        -
+        (\d+)   # 1 for staging, 3 for production
+    """,
+        re.VERBOSE,
+    )
+    _hostname = host_info.get("hostname")
+    if _hostname:
+        m = hostname_re.match(_hostname)
+        if m:
+            _function, _site, _num = m.groups()
+            host_info["function"] = _function
+            host_info["site"] = _site
+            if _num == "1":
+                host_info["environment"] = "staging"
+
+    # End local changes
+    return host_info
+
+
+def local_get_modules_hook(os_info: OSInfo, host_info: HostInfo, modules: Modules) -> Modules:
+    """Local hook to modify default set of modules in an OPS repo."""
+    # Start local changes in this repository
+
+    _eduid_modules = {
+        "apparmor",
+        "apt",
+        "augeas",
+        "bastion",
+        "concat",
+        "docker",
+        "munin",
+        "stdlib",
+        "sunet",
+        "ufw",
+    }
+    # Only keep the modules eduID actually uses
+    modules = {k: v for k, v in modules.items() if k in _eduid_modules}
+    logger.debug(f"Adding modules: {json.dumps(modules, sort_keys=True, indent=4)}")
+
+    # Use eduID tag for puppet-sunet
+    modules["sunet"]["tag"] = "eduid-stable-2*"
+    if host_info.get("environment") == "staging":
+        modules["sunet"]["tag"] = "eduid_dev-2*"
+
+    # use sunet_dev-2* for some modules in staging
+    for dev_module in ["munin"]:
+        if host_info.get("environment") == "staging" and dev_module in modules:
+            modules[dev_module]["tag"] = "sunet_dev-2*"
+
+    # End local changes
+    return modules
+
+
+def update_cosmos_modules(filename: str, modules: Modules) -> None:
+    """Create/update the cosmos-modules.conf file.
+
+    First, we check if the file already have the right content. If so, we do nothing.
+    """
+    content = "# This file is automatically generated by the setup_cosmos_modules script.\n# Do not edit it manually.\n"
+    for k, v in sorted(modules.items()):
+        content += f"{k:15} {v['repo']:55} {v['upgrade']:5} {v['tag']}\n"
+    _file = Path(filename)
+    if _file.exists():
+        # Check if the content is already correct, and avoid updating the file if so (so that the timestamp
+        # of the file at least indicates when the content was last updated)
+        with _file.open("r") as f:
+            current = f.read()
+        if current == content:
+            logger.debug(f"{filename} is up to date")
+            return
+
+    # Create/update the file by writing the content to a temporary file and then renaming it
+    _tmp_file = _file.with_suffix(".tmp")
+    with _tmp_file.open("w") as f:
+        f.write(content)
+    _tmp_file.rename(_file)
+    logger.debug(f"Updated {filename}")
+
+
+def _setup_logging(my_name: str, args: Arguments):
+    level = logging.INFO
+    if args.debug:
+        level = logging.DEBUG
+    logging.basicConfig(level=level, stream=sys.stderr, format="{asctime} | {levelname:7} | {message}", style="{")
+    global logger
+    logger = logging.getLogger(my_name)
+    # If stderr is not a TTY, change the log level of the StreamHandler (stream = sys.stderr above) to ERROR
+    if not sys.stderr.isatty() and not args.debug:
+        for this_h in logging.getLogger("").handlers:
+            this_h.setLevel(logging.ERROR)
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+
+
+def main(my_name: str, args: Arguments) -> bool:
+    _setup_logging(my_name, args)
+
+    os_info = get_os_info()
+    host_info = get_host_info()
+    modules = get_modules(os_info, host_info)
+
+    update_cosmos_modules(args.filename, modules)
+
+    return True
+
+
+if __name__ == "__main__":
+    my_name = os.path.basename(sys.argv[0])
+    args = parse_args()
+    res = main(my_name, args=args)
+    if res:
+        sys.exit(0)
+    sys.exit(1)