301 lines
11 KiB
Plaintext
301 lines
11 KiB
Plaintext
|
#!/usr/bin/env python3
|
||
|
|
#
|
||
|
|
# This script is responsible for creating/updating /etc/puppet/cosmos-modules.conf.
|
||
|
|
#
|
||
|
|
# If this script exits without creating that file, a default list of modules will be
|
||
|
|
# selected (by post-tasks.d/010cosmos-modules, the script that invokes this script).
|
||
|
|
#
|
||
|
|
# NOTES ABOUT THE IMPLEMENTATION:
|
||
|
|
#
|
||
|
|
# - Avoid any third party modules. We want this script to be re-usable in all ops-repos.
|
||
|
|
# - To make merging easier, try to keep all local alterations in the local_* functions.
|
||
|
|
# - Format with black and isort. Line width 120.
|
||
|
|
# - You probably ONLY want to change things in the local_get_modules_hook() function.
|
||
|
|
#
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import csv
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import logging.handlers
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import socket
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Dict, NewType, Optional, cast
|
||
|
|
|
||
|
|
from pkg_resources import parse_version
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__) # will be overwritten by _setup_logging()
|
||
|
|
|
||
|
|
# Set up types for data that is passed around in functions in this script.
|
||
|
|
# Need to use Dict (not dict) here since these aren't stripped by strip-hints, and doesn't work on Ubuntu <= 20.04.
|
||
|
|
Arguments = NewType("Arguments", argparse.Namespace)
|
||
|
|
OSInfo = Dict[str, str]
|
||
|
|
HostInfo = Dict[str, Optional[str]]
|
||
|
|
Modules = Dict[str, Dict[str, str]]
|
||
|
|
|
||
|
|
|
||
|
|
def parse_args() -> Arguments:
|
||
|
|
"""
|
||
|
|
Parse the command line arguments
|
||
|
|
"""
|
||
|
|
parser = argparse.ArgumentParser(
|
||
|
|
description="Setup cosmos-modules.conf",
|
||
|
|
add_help=True,
|
||
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||
|
|
)
|
||
|
|
|
||
|
|
parser.add_argument("--debug", dest="debug", action="store_true", default=False, help="Enable debug operation")
|
||
|
|
parser.add_argument(
|
||
|
|
"--filename", dest="filename", type=str, default="/etc/puppet/cosmos-modules.conf", help="Filename to write to"
|
||
|
|
)
|
||
|
|
|
||
|
|
return cast(Arguments, parser.parse_args())
|
||
|
|
|
||
|
|
|
||
|
|
def get_os_info() -> OSInfo:
|
||
|
|
"""Load info about the current OS (distro, release etc.)"""
|
||
|
|
os_info: OSInfo = {}
|
||
|
|
if Path("/etc/os-release").exists():
|
||
|
|
os_info.update({k.lower(): v for k, v in _parse_bash_vars("/etc/os-release").items()})
|
||
|
|
res = local_os_info_hook(os_info)
|
||
|
|
logger.debug(f"OS info:\n{json.dumps(res, sort_keys=True, indent=4)}")
|
||
|
|
return res
|
||
|
|
|
||
|
|
|
||
|
|
def get_host_info() -> HostInfo:
|
||
|
|
"""Load info about the current host (hostname, fqdn, domain name etc.)"""
|
||
|
|
try:
|
||
|
|
fqdn = socket.getfqdn()
|
||
|
|
hostname = socket.gethostname()
|
||
|
|
except OSError:
|
||
|
|
host_info = {}
|
||
|
|
else:
|
||
|
|
_domainname = fqdn[len(hostname + ".") :]
|
||
|
|
|
||
|
|
host_info: HostInfo = {
|
||
|
|
"domainname": _domainname,
|
||
|
|
"fqdn": fqdn,
|
||
|
|
"hostname": hostname,
|
||
|
|
}
|
||
|
|
res = local_host_info_hook(host_info)
|
||
|
|
logger.debug(f"Host info: {json.dumps(res, sort_keys=True, indent=4)}")
|
||
|
|
return res
|
||
|
|
|
||
|
|
|
||
|
|
def _parse_bash_vars(path: str) -> dict[str, str]:
|
||
|
|
"""
|
||
|
|
Parses a bash script and returns a dictionary representing the
|
||
|
|
variables declared in that script.
|
||
|
|
|
||
|
|
Source: https://dev.to/htv2012/how-to-parse-bash-variables-b4f
|
||
|
|
|
||
|
|
:param path: The path to the bash script
|
||
|
|
:return: Variables as a dictionary
|
||
|
|
"""
|
||
|
|
with open(path) as stream:
|
||
|
|
contents = stream.read().strip()
|
||
|
|
|
||
|
|
var_declarations = re.findall(r"^[a-zA-Z0-9_]+=.*$", contents, flags=re.MULTILINE)
|
||
|
|
reader = csv.reader(var_declarations, delimiter="=")
|
||
|
|
bash_vars = dict(reader)
|
||
|
|
return bash_vars
|
||
|
|
|
||
|
|
|
||
|
|
def get_modules(os_info: OSInfo, host_info: HostInfo) -> Modules:
|
||
|
|
"""Load the list of default modules.
|
||
|
|
|
||
|
|
This is more or less an inventory of all the modules we have. If you don't want
|
||
|
|
to use all modules in your OPS repo, you can filter them in the local hook.
|
||
|
|
|
||
|
|
If you want to use a different tag for a module on a specific host/os, you can
|
||
|
|
do that in the local hook as well.
|
||
|
|
"""
|
||
|
|
default_modules = """
|
||
|
|
# name repo upgrade tag
|
||
|
|
apparmor https://github.com/SUNET/puppet-apparmor.git yes sunet-2*
|
||
|
|
apt https://github.com/SUNET/puppetlabs-apt.git yes sunet-2*
|
||
|
|
augeas https://github.com/SUNET/puppet-augeas.git yes sunet-2*
|
||
|
|
bastion https://github.com/SUNET/puppet-bastion.git yes sunet-2*
|
||
|
|
concat https://github.com/SUNET/puppetlabs-concat.git yes sunet-2*
|
||
|
|
cosmos https://github.com/SUNET/puppet-cosmos.git yes sunet-2*
|
||
|
|
dhcp https://github.com/SUNET/puppetlabs-dhcp.git yes sunet_dev-2*
|
||
|
|
docker https://github.com/SUNET/garethr-docker.git yes sunet-2*
|
||
|
|
hiera-gpg https://github.com/SUNET/hiera-gpg.git yes sunet-2*
|
||
|
|
munin https://github.com/SUNET/ssm-munin.git yes sunet-2*
|
||
|
|
nagioscfg https://github.com/SUNET/puppet-nagioscfg.git yes sunet-2*
|
||
|
|
network https://github.com/SUNET/attachmentgenie-network.git yes sunet-2*
|
||
|
|
pound https://github.com/SUNET/puppet-pound.git yes sunet-2*
|
||
|
|
pyff https://github.com/samlbits/puppet-pyff.git yes puppet-pyff-*
|
||
|
|
python https://github.com/SUNET/puppet-python.git yes sunet-2*
|
||
|
|
stdlib https://github.com/SUNET/puppetlabs-stdlib.git yes sunet-2*
|
||
|
|
sunet https://github.com/SUNET/puppet-sunet.git yes sunet-2*
|
||
|
|
sysctl https://github.com/SUNET/puppet-sysctl.git yes sunet-2*
|
||
|
|
ufw https://github.com/SUNET/puppet-module-ufw.git yes sunet-2*
|
||
|
|
varnish https://github.com/samlbits/puppet-varnish.git yes puppet-varnish-*
|
||
|
|
vcsrepo https://github.com/SUNET/puppetlabs-vcsrepo.git yes sunet-2*
|
||
|
|
xinetd https://github.com/SUNET/puppetlabs-xinetd.git yes sunet-2*
|
||
|
|
"""
|
||
|
|
modules: Modules = {}
|
||
|
|
for line in default_modules.splitlines():
|
||
|
|
try:
|
||
|
|
if not line.strip() or line.strip().startswith("#"):
|
||
|
|
continue
|
||
|
|
_name, _url, _upgrade, _tag = line.split()
|
||
|
|
modules[_name] = {
|
||
|
|
"repo": _url,
|
||
|
|
"upgrade": _upgrade,
|
||
|
|
"tag": _tag,
|
||
|
|
}
|
||
|
|
except ValueError:
|
||
|
|
logger.error(f"Failed to parse line: {repr(line)}")
|
||
|
|
raise
|
||
|
|
|
||
|
|
# Remove the UFW module on Ubuntu >= 22.04 (nftables is used there instead)
|
||
|
|
if os_info.get("name") == "Ubuntu":
|
||
|
|
ver = os_info.get("version_id")
|
||
|
|
if ver:
|
||
|
|
if parse_version(ver) >= parse_version("22.04"):
|
||
|
|
logger.debug("Removing UFW module for Ubuntu >= 22.04")
|
||
|
|
del modules["ufw"]
|
||
|
|
else:
|
||
|
|
logger.debug("Keeping UFW module for Ubuntu < 22.04")
|
||
|
|
else:
|
||
|
|
logger.debug("Unknown Ubuntu module version, keeping UFW module")
|
||
|
|
|
||
|
|
return local_get_modules_hook(os_info, host_info, modules)
|
||
|
|
|
||
|
|
|
||
|
|
def local_os_info_hook(os_info: OSInfo) -> OSInfo:
|
||
|
|
"""Local hook to modify os_info in an OPS repo."""
|
||
|
|
# Start local changes in this repository
|
||
|
|
# End local changes
|
||
|
|
return os_info
|
||
|
|
|
||
|
|
|
||
|
|
def local_host_info_hook(host_info: HostInfo) -> HostInfo:
|
||
|
|
"""Local hook to modify host_info in an OPS repo."""
|
||
|
|
# Start local changes in this repository
|
||
|
|
|
||
|
|
# Regular expression to tease apart an eduID hostname
|
||
|
|
hostname_re = re.compile(
|
||
|
|
r"""^
|
||
|
|
(\w+) # function ('idp', 'apps', ...)
|
||
|
|
-
|
||
|
|
(\w+) # site ('tug', 'sthb', ...)
|
||
|
|
-
|
||
|
|
(\d+) # 1 for staging, 3 for production
|
||
|
|
""",
|
||
|
|
re.VERBOSE,
|
||
|
|
)
|
||
|
|
_hostname = host_info.get("hostname")
|
||
|
|
if _hostname:
|
||
|
|
m = hostname_re.match(_hostname)
|
||
|
|
if m:
|
||
|
|
_function, _site, _num = m.groups()
|
||
|
|
host_info["function"] = _function
|
||
|
|
host_info["site"] = _site
|
||
|
|
if _num == "1":
|
||
|
|
host_info["environment"] = "staging"
|
||
|
|
|
||
|
|
# End local changes
|
||
|
|
return host_info
|
||
|
|
|
||
|
|
|
||
|
|
def local_get_modules_hook(os_info: OSInfo, host_info: HostInfo, modules: Modules) -> Modules:
|
||
|
|
"""Local hook to modify default set of modules in an OPS repo."""
|
||
|
|
# Start local changes in this repository
|
||
|
|
|
||
|
|
_eduid_modules = {
|
||
|
|
"apparmor",
|
||
|
|
"apt",
|
||
|
|
"augeas",
|
||
|
|
"bastion",
|
||
|
|
"concat",
|
||
|
|
"docker",
|
||
|
|
"munin",
|
||
|
|
"stdlib",
|
||
|
|
"sunet",
|
||
|
|
"ufw",
|
||
|
|
}
|
||
|
|
# Only keep the modules eduID actually uses
|
||
|
|
modules = {k: v for k, v in modules.items() if k in _eduid_modules}
|
||
|
|
logger.debug(f"Adding modules: {json.dumps(modules, sort_keys=True, indent=4)}")
|
||
|
|
|
||
|
|
# Use eduID tag for puppet-sunet
|
||
|
|
modules["sunet"]["tag"] = "eduid-stable-2*"
|
||
|
|
if host_info.get("environment") == "staging":
|
||
|
|
modules["sunet"]["tag"] = "eduid_dev-2*"
|
||
|
|
|
||
|
|
# use sunet_dev-2* for some modules in staging
|
||
|
|
for dev_module in ["munin"]:
|
||
|
|
if host_info.get("environment") == "staging" and dev_module in modules:
|
||
|
|
modules[dev_module]["tag"] = "sunet_dev-2*"
|
||
|
|
|
||
|
|
# End local changes
|
||
|
|
return modules
|
||
|
|
|
||
|
|
|
||
|
|
def update_cosmos_modules(filename: str, modules: Modules) -> None:
|
||
|
|
"""Create/update the cosmos-modules.conf file.
|
||
|
|
|
||
|
|
First, we check if the file already have the right content. If so, we do nothing.
|
||
|
|
"""
|
||
|
|
content = "# This file is automatically generated by the setup_cosmos_modules script.\n# Do not edit it manually.\n"
|
||
|
|
for k, v in sorted(modules.items()):
|
||
|
|
content += f"{k:15} {v['repo']:55} {v['upgrade']:5} {v['tag']}\n"
|
||
|
|
_file = Path(filename)
|
||
|
|
if _file.exists():
|
||
|
|
# Check if the content is already correct, and avoid updating the file if so (so that the timestamp
|
||
|
|
# of the file at least indicates when the content was last updated)
|
||
|
|
with _file.open("r") as f:
|
||
|
|
current = f.read()
|
||
|
|
if current == content:
|
||
|
|
logger.debug(f"{filename} is up to date")
|
||
|
|
return
|
||
|
|
|
||
|
|
# Create/update the file by writing the content to a temporary file and then renaming it
|
||
|
|
_tmp_file = _file.with_suffix(".tmp")
|
||
|
|
with _tmp_file.open("w") as f:
|
||
|
|
f.write(content)
|
||
|
|
_tmp_file.rename(_file)
|
||
|
|
logger.debug(f"Updated {filename}")
|
||
|
|
|
||
|
|
|
||
|
|
def _setup_logging(my_name: str, args: Arguments):
|
||
|
|
level = logging.INFO
|
||
|
|
if args.debug:
|
||
|
|
level = logging.DEBUG
|
||
|
|
logging.basicConfig(level=level, stream=sys.stderr, format="{asctime} | {levelname:7} | {message}", style="{")
|
||
|
|
global logger
|
||
|
|
logger = logging.getLogger(my_name)
|
||
|
|
# If stderr is not a TTY, change the log level of the StreamHandler (stream = sys.stderr above) to ERROR
|
||
|
|
if not sys.stderr.isatty() and not args.debug:
|
||
|
|
for this_h in logging.getLogger("").handlers:
|
||
|
|
this_h.setLevel(logging.ERROR)
|
||
|
|
if args.debug:
|
||
|
|
logger.setLevel(logging.DEBUG)
|
||
|
|
|
||
|
|
|
||
|
|
def main(my_name: str, args: Arguments) -> bool:
|
||
|
|
_setup_logging(my_name, args)
|
||
|
|
|
||
|
|
os_info = get_os_info()
|
||
|
|
host_info = get_host_info()
|
||
|
|
modules = get_modules(os_info, host_info)
|
||
|
|
|
||
|
|
update_cosmos_modules(args.filename, modules)
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
my_name = os.path.basename(sys.argv[0])
|
||
|
|
args = parse_args()
|
||
|
|
res = main(my_name, args=args)
|
||
|
|
if res:
|
||
|
|
sys.exit(0)
|
||
|
|
sys.exit(1)
|