From c400bba97d8dae9c422937691ab1c174277fe77d Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 14:21:29 +0100 Subject: [PATCH 1/4] remove 'make db' The db-file, essentially providing reverse lookup of classes to host names, is only used by some Nagios configuration instances and causes continuing operational headaches in those ops-repos. It should be kept/refactored to only apply to the monitoring hosts in the cases where it is used, but we don't want any new ops-repos to use it hence it should be removed from upstream multiverse. --- fabfile/db.py | 49 ------------------------- global/overlay/etc/puppet/cosmos_enc.py | 35 ++++++++++++++---- 2 files changed, 27 insertions(+), 57 deletions(-) delete mode 100644 fabfile/db.py diff --git a/fabfile/db.py b/fabfile/db.py deleted file mode 100644 index 67b6645..0000000 --- a/fabfile/db.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import yaml -import re - -def _all_hosts(): - return filter(lambda fn: '.' in fn and not fn.startswith('.') and os.path.isdir(fn),os.listdir(".")) - -def _load_db(): - rules = dict() - rules_file = "cosmos-rules.yaml"; - if os.path.exists(rules_file): - with open(rules_file) as fd: - rules.update(yaml.load(fd)) - - all_hosts = _all_hosts() - - members = dict() - for node_name in all_hosts: - for reg,cls in rules.iteritems(): - if re.match(reg,node_name): - for cls_name in cls.keys(): - h = members.get(cls_name,[]) - h.append(node_name) - members[cls_name] = h - members['all'] = all_hosts - - classes = dict() - for node_name in all_hosts: - node_classes = dict() - for reg,cls in rules.iteritems(): - if re.match(reg,node_name): - node_classes.update(cls) - classes[node_name] = node_classes - - # Sort member lists for a more easy to read diff - for cls in members.keys(): - members[cls].sort() - - return dict(classes=classes,members=members) - -_db = None -def cosmos_db(): - global _db - if _db is None: - _db = _load_db() - return _db - -if __name__ == '__main__': - print yaml.dump(cosmos_db()) diff --git a/global/overlay/etc/puppet/cosmos_enc.py b/global/overlay/etc/puppet/cosmos_enc.py index 852fb25..dca12d3 100755 --- a/global/overlay/etc/puppet/cosmos_enc.py +++ b/global/overlay/etc/puppet/cosmos_enc.py @@ -1,18 +1,37 @@ #!/usr/bin/env python3 +# +# Puppet 'External Node Classifier' to tell puppet what classes to apply to this node. +# +# Docs: https://puppet.com/docs/puppet/5.3/nodes_external.html +# -import sys -import yaml import os import re +import sys + +import yaml + +rules_path = os.environ.get("COSMOS_RULES_PATH", "/etc/puppet") node_name = sys.argv[1] -db_file = os.environ.get("COSMOS_ENC_DB","/etc/puppet/cosmos-db.yaml") -db = dict(classes=dict()) +rules = dict() +for p in rules_path.split(":"): + rules_file = os.path.join(p, "cosmos-rules.yaml") + if os.path.exists(rules_file): + with open(rules_file) as fd: + rules.update(yaml.safe_load(fd)) -if os.path.exists(db_file): - with open(db_file) as fd: - db.update(yaml.load(fd)) +found = False +classes = dict() +for reg, cls in rules.items(): + if re.search(reg, node_name): + classes.update(cls) + found = True -print(yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[]))))) +if not found: + sys.stderr.write(f"{sys.argv[0]}: {node_name} not found in cosmos-rules.yaml\n") +print("---\n" + yaml.dump(dict(classes=classes))) + +sys.exit(0) From 1bddf21049e8ded369e6319de78adb0ad9b880f5 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 15:04:01 +0100 Subject: [PATCH 2/4] remove 'make db' target as well --- Makefile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Makefile b/Makefile index fac9f2b..a284f95 100644 --- a/Makefile +++ b/Makefile @@ -6,13 +6,7 @@ cosmos: upgrade: fab upgrade -db: global/overlay/etc/puppet/cosmos-db.yaml - -global/overlay/etc/puppet/cosmos-db.yaml: global/overlay/etc/puppet/cosmos-rules.yaml - @python ./fabfile/db.py > global/overlay/etc/puppet/cosmos-db.yaml - @git add global/overlay/etc/puppet/cosmos-db.yaml && git commit -m "update db" global/overlay/etc/puppet/cosmos-db.yaml - -tag: db +tag: ./bump-tag test_in_docker: From 252d478e2d309ade7c3bddf63bb3110a56712794 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 16:09:25 +0100 Subject: [PATCH 3/4] cosmos-db.yaml is no more --- host-puppet-conf-test | 1 - 1 file changed, 1 deletion(-) diff --git a/host-puppet-conf-test b/host-puppet-conf-test index e72008c..0844d08 100755 --- a/host-puppet-conf-test +++ b/host-puppet-conf-test @@ -24,7 +24,6 @@ then echo "Copying files to host..." rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@$HOSTNAME:/etc/puppet/cosmos-rules.yaml rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@$HOSTNAME:/etc/puppet/manifests/cosmos-site.pp - rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-db.yaml root@$HOSTNAME:/etc/puppet/cosmos-db.yaml rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@$HOSTNAME:/etc/hiera/data/common.yaml # Test if the user has symlinked puppet-sunet correctly From 496b9f4b31d02d101f7e5cf3bcee603ea199d02e Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Tue, 7 Feb 2023 16:09:37 +0100 Subject: [PATCH 4/4] shellcheck fixes --- host-puppet-conf-test | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/host-puppet-conf-test b/host-puppet-conf-test index 0844d08..2ddcbe9 100755 --- a/host-puppet-conf-test +++ b/host-puppet-conf-test @@ -17,14 +17,14 @@ PUPPET_ARGS=${PUPPET_ARGS-"--verbose"} # Check if cosmos or puppet is already running on host echo "Checking if puppet or cosmos is already running..." -ssh root@$HOSTNAME ps aux | egrep -v "grep|edit-secrets|gpg-agent" | egrep -q "cosmos|puppet" +ssh root@"$HOSTNAME" ps aux | grep -Ev "grep|edit-secrets|gpg-agent" | grep -Eq "cosmos|puppet" if [ $? -eq 1 ] then echo "Copying files to host..." - rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@$HOSTNAME:/etc/puppet/cosmos-rules.yaml - rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@$HOSTNAME:/etc/puppet/manifests/cosmos-site.pp - rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@$HOSTNAME:/etc/hiera/data/common.yaml + rsync -av --exclude '*~' global/overlay/etc/puppet/cosmos-rules.yaml root@"$HOSTNAME":/etc/puppet/cosmos-rules.yaml + rsync -av --exclude '*~' global/overlay/etc/puppet/manifests/cosmos-site.pp root@"$HOSTNAME":/etc/puppet/manifests/cosmos-site.pp + rsync -av --exclude '*~' global/overlay/etc/hiera/data/common.yaml root@"$HOSTNAME":/etc/hiera/data/common.yaml # Test if the user has symlinked puppet-sunet correctly # by first checking if the link exits and then whether @@ -36,7 +36,7 @@ then fi echo "Running puppet apply..." - ssh root@$HOSTNAME /usr/bin/puppet apply $PUPPET_ARGS /etc/puppet/manifests/cosmos-site.pp + ssh root@"$HOSTNAME" /usr/bin/puppet apply $PUPPET_ARGS /etc/puppet/manifests/cosmos-site.pp else echo "Cosmos or puppet already running. Exiting." exit 1