diff --git a/Makefile b/Makefile index 8936489..a284f95 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,7 @@ cosmos: upgrade: fab upgrade -db: - @python ./fabfile/db.py > global/overlay/etc/puppet/cosmos-db.yaml - @git add global/overlay/etc/puppet/cosmos-db.yaml && git commit -m "update db" global/overlay/etc/puppet/cosmos-db.yaml - -tag: db +tag: ./bump-tag test_in_docker: diff --git a/docs/cosmos-puppet-ops.mkd b/docs/cosmos-puppet-ops.mkd index 1d121b8..fc7fa9f 100644 --- a/docs/cosmos-puppet-ops.mkd +++ b/docs/cosmos-puppet-ops.mkd @@ -222,7 +222,7 @@ you'll try to push to the multiverse remote! Finally create a branch for the 'multiverse' upstream so you can merge changes to multiverse: ``` -# git checkout -b multiverse --track multiverse/master +# git checkout -b multiverse --track multiverse/main ``` Note that you can maintain your repo on just about any git hosting platform, including diff --git a/docs/setup_cosmos_modules.eduid.example b/docs/setup_cosmos_modules.eduid.example new file mode 100755 index 0000000..2b9dfdc --- /dev/null +++ b/docs/setup_cosmos_modules.eduid.example @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +# +# This script is responsible for creating/updating /etc/puppet/cosmos-modules.conf. +# +# If this script exits without creating that file, a default list of modules will be +# selected (by post-tasks.d/010cosmos-modules, the script that invokes this script). +# +# NOTES ABOUT THE IMPLEMENTATION: +# +# - Avoid any third party modules. We want this script to be re-usable in all ops-repos. +# - To make merging easier, try to keep all local alterations in the local_* functions. +# - Format with black and isort. Line width 120. +# - You probably ONLY want to change things in the local_get_modules_hook() function. +# + +import argparse +import csv +import json +import logging +import logging.handlers +import os +import re +import socket +import sys +from pathlib import Path +from typing import Dict, NewType, Optional, cast + +from pkg_resources import parse_version + +logger = logging.getLogger(__name__) # will be overwritten by _setup_logging() + +# Set up types for data that is passed around in functions in this script. +# Need to use Dict (not dict) here since these aren't stripped by strip-hints, and doesn't work on Ubuntu <= 20.04. +Arguments = NewType("Arguments", argparse.Namespace) +OSInfo = Dict[str, str] +HostInfo = Dict[str, Optional[str]] +Modules = Dict[str, Dict[str, str]] + + +def parse_args() -> Arguments: + """ + Parse the command line arguments + """ + parser = argparse.ArgumentParser( + description="Setup cosmos-modules.conf", + add_help=True, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument("--debug", dest="debug", action="store_true", default=False, help="Enable debug operation") + parser.add_argument( + "--filename", dest="filename", type=str, default="/etc/puppet/cosmos-modules.conf", help="Filename to write to" + ) + + return cast(Arguments, parser.parse_args()) + + +def get_os_info() -> OSInfo: + """Load info about the current OS (distro, release etc.)""" + os_info: OSInfo = {} + if Path("/etc/os-release").exists(): + os_info.update({k.lower(): v for k, v in _parse_bash_vars("/etc/os-release").items()}) + res = local_os_info_hook(os_info) + logger.debug(f"OS info:\n{json.dumps(res, sort_keys=True, indent=4)}") + return res + + +def get_host_info() -> HostInfo: + """Load info about the current host (hostname, fqdn, domain name etc.)""" + try: + fqdn = socket.getfqdn() + hostname = socket.gethostname() + except OSError: + host_info = {} + else: + _domainname = fqdn[len(hostname + ".") :] + + host_info: HostInfo = { + "domainname": _domainname, + "fqdn": fqdn, + "hostname": hostname, + } + res = local_host_info_hook(host_info) + logger.debug(f"Host info: {json.dumps(res, sort_keys=True, indent=4)}") + return res + + +def _parse_bash_vars(path: str) -> dict[str, str]: + """ + Parses a bash script and returns a dictionary representing the + variables declared in that script. + + Source: https://dev.to/htv2012/how-to-parse-bash-variables-b4f + + :param path: The path to the bash script + :return: Variables as a dictionary + """ + with open(path) as stream: + contents = stream.read().strip() + + var_declarations = re.findall(r"^[a-zA-Z0-9_]+=.*$", contents, flags=re.MULTILINE) + reader = csv.reader(var_declarations, delimiter="=") + bash_vars = dict(reader) + return bash_vars + + +def get_modules(os_info: OSInfo, host_info: HostInfo) -> Modules: + """Load the list of default modules. + + This is more or less an inventory of all the modules we have. If you don't want + to use all modules in your OPS repo, you can filter them in the local hook. + + If you want to use a different tag for a module on a specific host/os, you can + do that in the local hook as well. + """ + default_modules = """ + # name repo upgrade tag + apparmor https://github.com/SUNET/puppet-apparmor.git yes sunet-2* + apt https://github.com/SUNET/puppetlabs-apt.git yes sunet-2* + augeas https://github.com/SUNET/puppet-augeas.git yes sunet-2* + bastion https://github.com/SUNET/puppet-bastion.git yes sunet-2* + concat https://github.com/SUNET/puppetlabs-concat.git yes sunet-2* + cosmos https://github.com/SUNET/puppet-cosmos.git yes sunet-2* + dhcp https://github.com/SUNET/puppetlabs-dhcp.git yes sunet_dev-2* + docker https://github.com/SUNET/garethr-docker.git yes sunet-2* + hiera-gpg https://github.com/SUNET/hiera-gpg.git yes sunet-2* + munin https://github.com/SUNET/ssm-munin.git yes sunet-2* + nagioscfg https://github.com/SUNET/puppet-nagioscfg.git yes sunet-2* + network https://github.com/SUNET/attachmentgenie-network.git yes sunet-2* + pound https://github.com/SUNET/puppet-pound.git yes sunet-2* + pyff https://github.com/samlbits/puppet-pyff.git yes puppet-pyff-* + python https://github.com/SUNET/puppet-python.git yes sunet-2* + stdlib https://github.com/SUNET/puppetlabs-stdlib.git yes sunet-2* + sunet https://github.com/SUNET/puppet-sunet.git yes sunet-2* + sysctl https://github.com/SUNET/puppet-sysctl.git yes sunet-2* + ufw https://github.com/SUNET/puppet-module-ufw.git yes sunet-2* + varnish https://github.com/samlbits/puppet-varnish.git yes puppet-varnish-* + vcsrepo https://github.com/SUNET/puppetlabs-vcsrepo.git yes sunet-2* + xinetd https://github.com/SUNET/puppetlabs-xinetd.git yes sunet-2* + """ + modules: Modules = {} + for line in default_modules.splitlines(): + try: + if not line.strip() or line.strip().startswith("#"): + continue + _name, _url, _upgrade, _tag = line.split() + modules[_name] = { + "repo": _url, + "upgrade": _upgrade, + "tag": _tag, + } + except ValueError: + logger.error(f"Failed to parse line: {repr(line)}") + raise + + # Remove the UFW module on Ubuntu >= 22.04 (nftables is used there instead) + if os_info.get("name") == "Ubuntu": + ver = os_info.get("version_id") + if ver: + if parse_version(ver) >= parse_version("22.04"): + logger.debug("Removing UFW module for Ubuntu >= 22.04") + del modules["ufw"] + else: + logger.debug("Keeping UFW module for Ubuntu < 22.04") + else: + logger.debug("Unknown Ubuntu module version, keeping UFW module") + + return local_get_modules_hook(os_info, host_info, modules) + + +def local_os_info_hook(os_info: OSInfo) -> OSInfo: + """Local hook to modify os_info in an OPS repo.""" + # Start local changes in this repository + # End local changes + return os_info + + +def local_host_info_hook(host_info: HostInfo) -> HostInfo: + """Local hook to modify host_info in an OPS repo.""" + # Start local changes in this repository + + # Regular expression to tease apart an eduID hostname + hostname_re = re.compile( + r"""^ + (\w+) # function ('idp', 'apps', ...) + - + (\w+) # site ('tug', 'sthb', ...) + - + (\d+) # 1 for staging, 3 for production + """, + re.VERBOSE, + ) + _hostname = host_info.get("hostname") + if _hostname: + m = hostname_re.match(_hostname) + if m: + _function, _site, _num = m.groups() + host_info["function"] = _function + host_info["site"] = _site + if _num == "1": + host_info["environment"] = "staging" + + # End local changes + return host_info + + +def local_get_modules_hook(os_info: OSInfo, host_info: HostInfo, modules: Modules) -> Modules: + """Local hook to modify default set of modules in an OPS repo.""" + # Start local changes in this repository + + _eduid_modules = { + "apparmor", + "apt", + "augeas", + "bastion", + "concat", + "docker", + "munin", + "stdlib", + "sunet", + "ufw", + } + # Only keep the modules eduID actually uses + modules = {k: v for k, v in modules.items() if k in _eduid_modules} + logger.debug(f"Adding modules: {json.dumps(modules, sort_keys=True, indent=4)}") + + # Use eduID tag for puppet-sunet + modules["sunet"]["tag"] = "eduid-stable-2*" + if host_info.get("environment") == "staging": + modules["sunet"]["tag"] = "eduid_dev-2*" + + # use sunet_dev-2* for some modules in staging + for dev_module in ["munin"]: + if host_info.get("environment") == "staging" and dev_module in modules: + modules[dev_module]["tag"] = "sunet_dev-2*" + + # End local changes + return modules + + +def update_cosmos_modules(filename: str, modules: Modules) -> None: + """Create/update the cosmos-modules.conf file. + + First, we check if the file already have the right content. If so, we do nothing. + """ + content = "# This file is automatically generated by the setup_cosmos_modules script.\n# Do not edit it manually.\n" + for k, v in sorted(modules.items()): + content += f"{k:15} {v['repo']:55} {v['upgrade']:5} {v['tag']}\n" + _file = Path(filename) + if _file.exists(): + # Check if the content is already correct, and avoid updating the file if so (so that the timestamp + # of the file at least indicates when the content was last updated) + with _file.open("r") as f: + current = f.read() + if current == content: + logger.debug(f"{filename} is up to date") + return + + # Create/update the file by writing the content to a temporary file and then renaming it + _tmp_file = _file.with_suffix(".tmp") + with _tmp_file.open("w") as f: + f.write(content) + _tmp_file.rename(_file) + logger.debug(f"Updated {filename}") + + +def _setup_logging(my_name: str, args: Arguments): + level = logging.INFO + if args.debug: + level = logging.DEBUG + logging.basicConfig(level=level, stream=sys.stderr, format="{asctime} | {levelname:7} | {message}", style="{") + global logger + logger = logging.getLogger(my_name) + # If stderr is not a TTY, change the log level of the StreamHandler (stream = sys.stderr above) to ERROR + if not sys.stderr.isatty() and not args.debug: + for this_h in logging.getLogger("").handlers: + this_h.setLevel(logging.ERROR) + if args.debug: + logger.setLevel(logging.DEBUG) + + +def main(my_name: str, args: Arguments) -> bool: + _setup_logging(my_name, args) + + os_info = get_os_info() + host_info = get_host_info() + modules = get_modules(os_info, host_info) + + update_cosmos_modules(args.filename, modules) + + return True + + +if __name__ == "__main__": + my_name = os.path.basename(sys.argv[0]) + args = parse_args() + res = main(my_name, args=args) + if res: + sys.exit(0) + sys.exit(1) diff --git a/docs/setup_cosmos_modules.example b/docs/setup_cosmos_modules.example index 9b4c7e2..6b1b9c6 100755 --- a/docs/setup_cosmos_modules.example +++ b/docs/setup_cosmos_modules.example @@ -1,134 +1,216 @@ #!/usr/bin/env python3 +""" Write out a puppet cosmos-modules.conf """ + +import hashlib +import os +import os.path +import sys try: from configobj import ConfigObj - os_info = ConfigObj("/etc/os-release") + OS_INFO = ConfigObj("/etc/os-release") except (IOError, ModuleNotFoundError): - os_info = None + OS_INFO = None -modulesfile: str = "/etc/puppet/cosmos-modules.conf" -modules: dict = { - "concat": { - "repo": "https://github.com/SUNET/puppetlabs-concat.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "stdlib": { - "repo": "https://github.com/SUNET/puppetlabs-stdlib.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "cosmos": { - "repo": "https://github.com/SUNET/puppet-cosmos.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "ufw": { - "repo": "https://github.com/SUNET/puppet-module-ufw.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "apt": { - "repo": "https://github.com/SUNET/puppetlabs-apt.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "vcsrepo": { - "repo": "https://github.com/SUNET/puppetlabs-vcsrepo.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "xinetd": { - "repo": "https://github.com/SUNET/puppetlabs-xinetd.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "python": { - "repo": "https://github.com/SUNET/puppet-python.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "hiera-gpg": { - "repo": "https://github.com/SUNET/hiera-gpg.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "pound": { - "repo": "https://github.com/SUNET/puppet-pound.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "augeas": { - "repo": "https://github.com/SUNET/puppet-augeas.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "bastion": { - "repo": "https://github.com/SUNET/puppet-bastion.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "pyff": { - "repo": "https://github.com/samlbits/puppet-pyff.git", - "upgrade": "yes", - "tag": "puppet-pyff-*", - }, - "dhcp": { - "repo": "https://github.com/SUNET/puppetlabs-dhcp.git", - "upgrade": "yes", - "tag": "sunet_dev-2*", - }, - "varnish": { - "repo": "https://github.com/samlbits/puppet-varnish.git", - "upgrade": "yes", - "tag": "puppet-varnish-*", - }, - "apparmor": { - "repo": "https://github.com/SUNET/puppet-apparmor.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "docker": { - "repo": "https://github.com/SUNET/garethr-docker.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "network": { - "repo": "https://github.com/SUNET/attachmentgenie-network.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "sunet": { - "repo": "https://github.com/SUNET/puppet-sunet.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "sysctl": { - "repo": "https://github.com/SUNET/puppet-sysctl.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, - "nagioscfg": { - "repo": "https://github.com/SUNET/puppet-nagioscfg.git", - "upgrade": "yes", - "tag": "sunet-2*", - }, -} +def get_file_hash(modulesfile): + """ + Based on https://github.com/python/cpython/pull/31930: should use + hashlib.file_digest() but it is only available in python 3.11 + """ + try: + with open(modulesfile, "rb") as fileobj: + digestobj = hashlib.sha256() + _bufsize = 2**18 + buf = bytearray(_bufsize) # Reusable buffer to reduce allocations. + view = memoryview(buf) + while True: + size = fileobj.readinto(buf) + if size == 0: + break # EOF + digestobj.update(view[:size]) + except FileNotFoundError: + return "" -# When/if we want we can do stuff to modules here -if os_info: - if os_info["VERSION_CODENAME"] == "bullseye": - pass + return digestobj.hexdigest() -with open(modulesfile, "w") as fh: + +def get_list_hash(file_lines): + """Get hash of list contents""" + + file_lines_hash = hashlib.sha256() + for line in file_lines: + file_lines_hash.update(line) + + return file_lines_hash.hexdigest() + + +def create_file_content(modules): + """ + Write out the expected file contents to a list so we can check the + expected checksum before writing anything + """ + file_lines = [] + file_lines.append( + "# Generated by {}\n".format( # pylint: disable=consider-using-f-string + os.path.basename(sys.argv[0]) + ).encode("utf-8") + ) for key in modules: - fh.write( - "{0:11} {1} {2} {3}\n".format( + file_lines.append( + "{0:11} {1} {2} {3}\n".format( # pylint: disable=consider-using-f-string key, modules[key]["repo"], modules[key]["upgrade"], modules[key]["tag"], - ) + ).encode("utf-8") ) + + return file_lines + + +def main(): + """Starting point of the program""" + + modulesfile: str = "/etc/puppet/cosmos-modules.conf" + modulesfile_tmp: str = modulesfile + ".tmp" + + modules: dict = { + "concat": { + "repo": "https://github.com/SUNET/puppetlabs-concat.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "stdlib": { + "repo": "https://github.com/SUNET/puppetlabs-stdlib.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "cosmos": { + "repo": "https://github.com/SUNET/puppet-cosmos.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "ufw": { + "repo": "https://github.com/SUNET/puppet-module-ufw.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "apt": { + "repo": "https://github.com/SUNET/puppetlabs-apt.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "vcsrepo": { + "repo": "https://github.com/SUNET/puppetlabs-vcsrepo.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "xinetd": { + "repo": "https://github.com/SUNET/puppetlabs-xinetd.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "python": { + "repo": "https://github.com/SUNET/puppet-python.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "hiera-gpg": { + "repo": "https://github.com/SUNET/hiera-gpg.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "pound": { + "repo": "https://github.com/SUNET/puppet-pound.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "augeas": { + "repo": "https://github.com/SUNET/puppet-augeas.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "bastion": { + "repo": "https://github.com/SUNET/puppet-bastion.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "pyff": { + "repo": "https://github.com/samlbits/puppet-pyff.git", + "upgrade": "yes", + "tag": "puppet-pyff-*", + }, + "dhcp": { + "repo": "https://github.com/SUNET/puppetlabs-dhcp.git", + "upgrade": "yes", + "tag": "sunet_dev-2*", + }, + "varnish": { + "repo": "https://github.com/samlbits/puppet-varnish.git", + "upgrade": "yes", + "tag": "puppet-varnish-*", + }, + "apparmor": { + "repo": "https://github.com/SUNET/puppet-apparmor.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "docker": { + "repo": "https://github.com/SUNET/garethr-docker.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "network": { + "repo": "https://github.com/SUNET/attachmentgenie-network.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "sunet": { + "repo": "https://github.com/SUNET/puppet-sunet.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "sysctl": { + "repo": "https://github.com/SUNET/puppet-sysctl.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + "nagioscfg": { + "repo": "https://github.com/SUNET/puppet-nagioscfg.git", + "upgrade": "yes", + "tag": "sunet-2*", + }, + } + + # When/if we want we can do stuff to modules here + if OS_INFO: + if OS_INFO["VERSION_CODENAME"] == "bullseye": + pass + + # Build list of expected file content + file_lines = create_file_content(modules) + + # Get hash of the list + list_hash = get_list_hash(file_lines) + + # Get hash of the existing file on disk + file_hash = get_file_hash(modulesfile) + + # Update the file if necessary + if list_hash != file_hash: + # Since we are reading the file with 'rb' when computing our hash use 'wb' when + # writing so we dont end up creating a file that does not match the + # expected hash + with open(modulesfile_tmp, "wb") as fileobj: + for line in file_lines: + fileobj.write(line) + + # Rename it in place so the update is atomic for anything else trying to + # read the file + os.rename(modulesfile_tmp, modulesfile) + + +if __name__ == "__main__": + main() diff --git a/edit-secrets b/edit-secrets index b4d816c..a2c67ac 100755 --- a/edit-secrets +++ b/edit-secrets @@ -151,7 +151,7 @@ function edit_file_on_host() { edit_gpg_file ${SECRETFILE} elif [ -f /etc/hiera/eyaml/public_certkey.pkcs7.pem ]; then # default to eyaml if the key exists and none of the secrets-file above exist - touch ${EYAMLFILE} + echo "---" > ${EYAMLFILE} edit_eyaml_file ${EYAMLFILE} fi } diff --git a/fabfile/db.py b/fabfile/db.py deleted file mode 100644 index 67b6645..0000000 --- a/fabfile/db.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import yaml -import re - -def _all_hosts(): - return filter(lambda fn: '.' in fn and not fn.startswith('.') and os.path.isdir(fn),os.listdir(".")) - -def _load_db(): - rules = dict() - rules_file = "cosmos-rules.yaml"; - if os.path.exists(rules_file): - with open(rules_file) as fd: - rules.update(yaml.load(fd)) - - all_hosts = _all_hosts() - - members = dict() - for node_name in all_hosts: - for reg,cls in rules.iteritems(): - if re.match(reg,node_name): - for cls_name in cls.keys(): - h = members.get(cls_name,[]) - h.append(node_name) - members[cls_name] = h - members['all'] = all_hosts - - classes = dict() - for node_name in all_hosts: - node_classes = dict() - for reg,cls in rules.iteritems(): - if re.match(reg,node_name): - node_classes.update(cls) - classes[node_name] = node_classes - - # Sort member lists for a more easy to read diff - for cls in members.keys(): - members[cls].sort() - - return dict(classes=classes,members=members) - -_db = None -def cosmos_db(): - global _db - if _db is None: - _db = _load_db() - return _db - -if __name__ == '__main__': - print yaml.dump(cosmos_db()) diff --git a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh index c31cea7..5e27f3d 100755 --- a/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh +++ b/global/overlay/etc/cosmos/apt/bootstrap-cosmos.sh @@ -30,7 +30,7 @@ export DEBIAN_FRONTEND='noninteractive' apt-get -y update apt-get -y upgrade -for pkg in rsync git git-core wget gpg; do +for pkg in rsync git git-core wget gpg jq; do # script is running with "set -e", use "|| true" to allow packages to not # exist without stopping the script apt-get -y install $pkg || true @@ -56,16 +56,43 @@ mv -f /etc/rc.local.new /etc/rc.local touch /etc/run-cosmos-at-boot # If this cloud-config is set, it will interfere with our changes to /etc/hosts -if [ -f /etc/cloud/cloud.cfg ]; then - sed -i 's/manage_etc_hosts: true/manage_etc_hosts: false/g' /etc/cloud/cloud.cfg -fi +# The configuration seems to move around between cloud-config versions +for file in /etc/cloud/cloud.cfg /etc/cloud/cloud.cfg.d/01_debian_cloud.cfg; do + if [ -f ${file} ]; then + sed -i 's/manage_etc_hosts: true/manage_etc_hosts: false/g' ${file} + fi +done -# Remove potential $hostname.novalocal line from /etc/hosts, added by cloud-init -sed -i.bak -e "s/^127\.0\.1\.1 $(hostname)\..*novalocal.*//1" /etc/hosts +# Remove potential $hostname.novalocal, added by cloud-init or Debian default +# from /etc/hosts. We add our own further down. +# +# From # https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution: +# "For a system with a permanent IP address, that permanent IP address should +# be used here instead of 127.0.1.1." +sed -i.bak -e "/127\.0\.1\.1/d" /etc/hosts + +vendor=$(lsb_release -is) +version=$(lsb_release -rs) +min_version=1337 +host_ip=127.0.1.1 +if [ "${vendor}" = "Ubuntu" ]; then + min_version=20.04 +elif [ "${vendor}" = "Debian" ]; then + min_version=11 +fi hostname $cmd_hostname short=`echo ${cmd_hostname} | awk -F. '{print $1}'` -echo "127.0.1.1 ${cmd_hostname} ${short}" >> /etc/hosts +# Only change behavior on modern OS where `ip -j` outputs a json predictuble +# enought to work with. +# +# Use `dpkg` to easier compare ubuntu versions. +if dpkg --compare-versions "${version}" "ge" "${min_version}"; then + # When hostname pointed to loopback in /etc/hosts containers running on the + # host tried to connect to the container itself instead of the host. + host_ip=$(ip -j address show "$(ip -j route show default | jq -r '.[0].dev')" | jq -r .[0].addr_info[0].local) +fi +echo "${host_ip} ${cmd_hostname} ${short}" >> /etc/hosts # Set up cosmos models. They are in the order of most significant first, so we want # diff --git a/global/overlay/etc/cron.d/cosmos b/global/overlay/etc/cron.d/cosmos index 4eab8de..3840f8c 100644 --- a/global/overlay/etc/cron.d/cosmos +++ b/global/overlay/etc/cron.d/cosmos @@ -1,4 +1,6 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -*/15 * * * * root test -f /etc/no-automatic-cosmos || /usr/local/bin/run-cosmos +*/15 * * * * root /usr/local/libexec/cosmos-cron-wrapper + +@reboot root sleep 30; /usr/local/libexec/cosmos-cron-wrapper diff --git a/global/overlay/etc/logrotate.d/docker-containers b/global/overlay/etc/logrotate.d/docker-containers deleted file mode 100644 index e9c90b8..0000000 --- a/global/overlay/etc/logrotate.d/docker-containers +++ /dev/null @@ -1,7 +0,0 @@ -/var/lib/docker/containers/*/*.log { - rotate 7 - daily - compress - delaycompress - copytruncate -} diff --git a/global/overlay/etc/puppet/cosmos_enc.py b/global/overlay/etc/puppet/cosmos_enc.py index 852fb25..dca12d3 100755 --- a/global/overlay/etc/puppet/cosmos_enc.py +++ b/global/overlay/etc/puppet/cosmos_enc.py @@ -1,18 +1,37 @@ #!/usr/bin/env python3 +# +# Puppet 'External Node Classifier' to tell puppet what classes to apply to this node. +# +# Docs: https://puppet.com/docs/puppet/5.3/nodes_external.html +# -import sys -import yaml import os import re +import sys + +import yaml + +rules_path = os.environ.get("COSMOS_RULES_PATH", "/etc/puppet") node_name = sys.argv[1] -db_file = os.environ.get("COSMOS_ENC_DB","/etc/puppet/cosmos-db.yaml") -db = dict(classes=dict()) +rules = dict() +for p in rules_path.split(":"): + rules_file = os.path.join(p, "cosmos-rules.yaml") + if os.path.exists(rules_file): + with open(rules_file) as fd: + rules.update(yaml.safe_load(fd)) -if os.path.exists(db_file): - with open(db_file) as fd: - db.update(yaml.load(fd)) +found = False +classes = dict() +for reg, cls in rules.items(): + if re.search(reg, node_name): + classes.update(cls) + found = True -print(yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[]))))) +if not found: + sys.stderr.write(f"{sys.argv[0]}: {node_name} not found in cosmos-rules.yaml\n") +print("---\n" + yaml.dump(dict(classes=classes))) + +sys.exit(0) diff --git a/global/overlay/etc/puppet/hiera.yaml b/global/overlay/etc/puppet/hiera.yaml index 3663305..3de986b 100644 --- a/global/overlay/etc/puppet/hiera.yaml +++ b/global/overlay/etc/puppet/hiera.yaml @@ -1,21 +1,27 @@ +# Hiera version 5 configuration +# --- -:backends: - - yaml - - gpg +version: 5 +defaults: + datadir: /etc/hiera/data + data_hash: yaml_data -:logger: console +hierarchy: + - name: "Per-node data" + path: "local.yaml" -:hierarchy: - - "%{env}/%{location}/%{calling_module}" - - "%{env}/%{calling_module}" - - local - - secrets.yaml - - common + - name: "Per-group data" + path: "group.yaml" + - name: "Per-host secrets" + path: "local.eyaml" + lookup_key: eyaml_lookup_key + options: + pkcs7_private_key: /etc/hiera/eyaml/private_key.pkcs7.pem + pkcs7_public_key: /etc/hiera/eyaml/public_certkey.pkcs7.pem -:yaml: - :datadir: /etc/hiera/data + - name: "Overrides per distribution" + path: "dist_%{::lsbdistcodename}_override.yaml" -:gpg: - :datadir: /etc/hiera/data - :key_dir: /etc/hiera/gpg + - name: "Data common to whole environment" + path: "common.yaml" \ No newline at end of file diff --git a/global/overlay/usr/local/bin/run-cosmos b/global/overlay/usr/local/bin/run-cosmos index 5f2cbc1..fdfb85d 100755 --- a/global/overlay/usr/local/bin/run-cosmos +++ b/global/overlay/usr/local/bin/run-cosmos @@ -16,29 +16,29 @@ lock() { eval "exec $fd>$lock_file" # acquier the lock - flock -n $fd \ + flock -n "$fd" \ && return 0 \ || return 1 } eexit() { - local error_str="$@" + local error_str="$*" - echo $error_str + echo "$error_str" exit 1 } main () { - lock $PROGNAME || eexit "Only one instance of $PROGNAME can run at one time." - cosmos $* update - cosmos $* apply + lock "$PROGNAME" || eexit "Only one instance of $PROGNAME can run at one time." + cosmos "$@" update + cosmos "$@" apply touch /var/run/last-cosmos-ok.stamp - find /var/lib/puppet/reports/ -type f -mtime +10 | xargs rm -f + find /var/lib/puppet/reports/ -type f -mtime +10 -print0 | xargs -0 rm -f } -main $* +main "$@" if [ -f /cosmos-reboot ]; then rm -f /cosmos-reboot diff --git a/global/overlay/usr/local/libexec/cosmos-cron-wrapper b/global/overlay/usr/local/libexec/cosmos-cron-wrapper new file mode 100755 index 0000000..ae66810 --- /dev/null +++ b/global/overlay/usr/local/libexec/cosmos-cron-wrapper @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +test -f /etc/no-automatic-cosmos && exit 0 + +RUN_COSMOS='/usr/local/bin/run-cosmos' +SCRIPTHERDER_CMD='' + +if [ -x /usr/local/bin/scriptherder ]; then + SCRIPTHERDER_CMD='/usr/local/bin/scriptherder --mode wrap --syslog --name cosmos --' +fi + +exec ${SCRIPTHERDER_CMD} ${RUN_COSMOS} "$@" diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index ee1889f..39569b2 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -42,7 +42,10 @@ if [ -f $CONFIG -o $LOCALCONFIG ]; then if [ "$src" != "$(git config remote.origin.url)" ]; then git config remote.origin.url $src fi - git pull -q + # Support master branch being renamed to main + git branch --all | grep -q '^[[:space:]]*remotes/origin/main$' && git checkout main + # Update repo and clean out any local inconsistencies + git pull -q || (git fetch && git reset --hard) else continue fi diff --git a/global/post-tasks.d/030puppet b/global/post-tasks.d/030puppet index af45005..561ebc4 100755 --- a/global/post-tasks.d/030puppet +++ b/global/post-tasks.d/030puppet @@ -1,13 +1,15 @@ #!/bin/sh +set -e + if [ "x$COSMOS_VERBOSE" = "xy" ]; then args="--verbose --show_diff" else args="--logdest=syslog" fi -if [ -f /usr/bin/puppet -a -d /etc/puppet/manifests ]; then - for m in `find /etc/puppet/manifests -name \*.pp`; do +if [ -f /usr/bin/puppet ] && [ -d /etc/puppet/manifests ]; then + find /etc/puppet/manifests -name \*.pp | while read -r m; do test "x$COSMOS_VERBOSE" = "xy" && echo "$0: Applying Puppet manifest $m" puppet apply $args $m done diff --git a/global/pre-tasks.d/015set-overlay-permissions b/global/pre-tasks.d/015set-overlay-permissions index 373ef68..37f9844 100755 --- a/global/pre-tasks.d/015set-overlay-permissions +++ b/global/pre-tasks.d/015set-overlay-permissions @@ -15,5 +15,9 @@ if ! test -d "$MODEL_OVERLAY"; then fi if [ -d "$MODEL_OVERLAY/root" ]; then - chmod -v 0700 "$MODEL_OVERLAY"/root + args="" + if [ "x$COSMOS_VERBOSE" = "xy" ]; then + args="-v" + fi + chmod ${args} 0700 "$MODEL_OVERLAY"/root fi diff --git a/global/pre-tasks.d/040hiera-eyaml b/global/pre-tasks.d/040hiera-eyaml new file mode 100755 index 0000000..1f2758d --- /dev/null +++ b/global/pre-tasks.d/040hiera-eyaml @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Set up eyaml for Hiera +# + +set -e + +EYAMLDIR=/etc/hiera/eyaml + +vendor=$(lsb_release -is) +version=$(lsb_release -rs) +# eyaml is only used on Ubuntu 20.04 and newer, and Debian 11 and newer (earlier OSes use hiera-gpg instead) +test "${vendor}" = "Ubuntu" && dpkg --compare-versions "${version}" "lt" "18.04" && exit 0 +test "${vendor}" = "Debian" && dpkg --compare-versions "${version}" "lt" "10" && exit 0 + +stamp="$COSMOS_BASE/stamps/hiera-eyaml-v01.stamp" + +test -f "$stamp" && exit 0 + +if [ ! -f /usr/bin/eyaml ] || [ ! -d /usr/share/doc/yaml-mode ]; then + apt-get update + apt-get -y install hiera-eyaml yaml-mode +fi + +if [ ! -f ${EYAMLDIR}/public_certkey.pkcs7.pem ] || [ ! -f ${EYAMLDIR}/private_key.pkcs7.pem ]; then + # hiera-eyaml wants a certificate and public key, not just a public key oddly enough + echo "$0: Generating eyaml key in ${EYAMLDIR} - this might take a while..." + mkdir -p /etc/hiera/eyaml + openssl req -x509 -newkey rsa:4096 -keyout ${EYAMLDIR}/private_key.pkcs7.pem \ + -out ${EYAMLDIR}/public_certkey.pkcs7.pem -days 3653 -nodes -sha256 \ + -subj "/C=SE/O=SUNET/OU=EYAML/CN=$(hostname)" + rm -f ${EYAMLDIR}/public_key.pkcs7.pem # cleanup +fi + +mkdir -p "$(dirname "${stamp}")" +touch "$stamp" diff --git a/global/pre-tasks.d/040hiera-gpg b/global/pre-tasks.d/040hiera-gpg index e5de6da..bc1da35 100755 --- a/global/pre-tasks.d/040hiera-gpg +++ b/global/pre-tasks.d/040hiera-gpg @@ -9,12 +9,21 @@ set -e GNUPGHOME=/etc/hiera/gpg export GNUPGHOME +vendor=$(lsb_release -is) +version=$(lsb_release -rs) +# If the OS is Ubuntu 18.04 or newer, or Debian 10 or newer, we don't need to do anything (those use eyaml instead) +test "${vendor}" = "Ubuntu" && dpkg --compare-versions "${version}" "ge" "18.04" && exit 0 +test "${vendor}" = "Debian" && dpkg --compare-versions "${version}" "ge" "10" && exit 0 + +stamp="$COSMOS_BASE/stamps/hiera-gpg-v01.stamp" + +test -f "$stamp" && exit 0 + if [ ! -f /usr/lib/ruby/vendor_ruby/gpgme.rb ]; then apt-get update apt-get -y install ruby-gpgme fi - if [ ! -s $GNUPGHOME/secring.gpg ]; then if [ "x$1" != "x--force" ]; then @@ -35,18 +44,21 @@ if [ ! -s $GNUPGHOME/secring.gpg ]; then chmod 700 $GNUPGHOME TMPFILE=$(mktemp /tmp/hiera-gpg.XXXXXX) - cat > $TMPFILE < "$TMPFILE" <