added facts

This commit is contained in:
Maria Haider 2024-05-13 18:50:11 +02:00
parent d4799d1139
commit 4e008b462a
Signed by: mariah
GPG key ID: 7414A760CA747E57
7 changed files with 866 additions and 0 deletions

View file

@ -0,0 +1,26 @@
#
# This file was created by ./scripts/cosmos-facts.
#
# 1 roles defined in metadata/roles-in.yaml
# 54 groups formed in the current Cosmos environment
#
# Roles <- groups:
# frontend_server <- fe
#
# Groups -> roles:
# fe -> frontend_server
#
# Unused groups: auditd, autoupdate, common, country, dockerhost2, eid_de_client, eid_dockerhost, eid_idm_app, eid_idm_db, eid_idm_redis, eid_kvmhost, eid_nagios_monitor, eidas_hsm_client, eidas_metadata_key, entropyclient, eumd, eupub, fe_servers, frontend_load_balancer, frontend_register_sites, idm, infra_ca_rp, invent_client, jump, jumphosts, konsulter, kvmdemw, kvmeidas, kvmfe, kvminfra, kvmmeta, mailclient, md, md_publisher, md_repo_client, md_repo_server, md_signer, mdsl_publisher, metadata_mdq_publisher, metadata_mdqp, metadata_metadata_repo, metadata_pyff_compose, monitor, natmd, natpub, nrpe, p, r, rsyslog, server, sunet_iaas_cloud, sunetops, validator
#
---
cosmos:
frontend_server_addrs:
- 94.176.224.165
- 94.176.224.166
- 94.176.224.37
- 94.176.224.38
frontend_server_hosts:
- fe-fre-1.test.komreg.net
- fe-fre-3.komreg.net
- fe-tug-1.test.komreg.net
- fe-tug-3.komreg.net

6
metadata/roles-in.yaml Normal file
View file

@ -0,0 +1,6 @@
---
cosmos:
roles:
frontend_server:
groups: [fe]

441
scripts/cosmos-facts Executable file
View file

@ -0,0 +1,441 @@
#!/usr/bin/env python3
#
# Copyright 2017 SUNET. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY SUNET ``AS IS'' AND ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SUNET OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are those of the
# authors and should not be interpreted as representing official policies, either expressed
# or implied, of SUNET.
#
# Author : Fredrik Thulin <fredrik@thulin.net>
#
#
# TL; DR: This script produces a Puppet fact called 'cosmos', with information
# about hosts in the current Cosmos environment:
#
# $facts['cosmos']['mongodb_server_addrs'] = [130.242.130.220, 130.242.130.221,
# '2001:6b0:54:c3:5054:ff:fea0:1dc', '2001:6b0:54:c3:5054:ff:fea0:1dd']
#
# $facts['cosmos']['mongodb_server_hosts'] = [userdb-fre-1.eduid.se, userdb-tug-1.eduid.se]
#
#
# This script creates the $facts['cosmos'] Puppet fact. This fact holds
# information about the Cosmos environment. To start with, this fact is
# used to define roles of hosts in the Cosmos environment.
#
# An example to provide the merits for this fact, and how it works:
#
# eduID has the two applications 'signup' and 'dashboard'. They are
# quite common and both have access to MongoDB, both register with
# frontends etc.
#
# eduID had lots of Hiera data like this: (different values in test
# and production, making it harder than necessary to deploy things
# from testing to production):
#
# eduid_frontend_servers:
# - fe-tug-1.eduid.se
# - fe-fre-1.eduid.se
#
# eduid_frontend_ips:
# - 130.242.131.3
# - 130.242.131.4
# - 2001:6b0:54:c4::3
# - 2001:6b0:54:c4::4
#
# eduid_mongodb_servers:
# - userdb-tug-1.eduid.se
# - userdb-fre-1.eduid.se
# - userdb-tug-2.eduid.se
#
# eduid_signup_haproxy_backends:
# - signup-tug-1.eduid.se
# #- signup-fre-1.eduid.se
#
# eduid_dashboard_haproxy_backends:
# - dash-tug-1.eduid.se
# - dash-fre-1.eduid.se
#
# eduid_signup_ips:
# - 130.242.130.212 # signup-tug-1.eduid.se
# - 2001:6b0:54:c3:5054:ff:fea0:1d4 # signup-tug-1.eduid.se
# eduid_dashboard_ips:
# - 130.242.130.213 # dash-fre-1.eduid.se
# - 130.242.130.197 # dash-tug-1.eduid.se
# - 2001:6b0:54:c3:5054:ff:fea0:1d5 # dash-fre-1.eduid.se
# - 2001:6b0:54:c3:5054:ff:fea0:1c5 # dash-tug-1.eduid.se
#
# This probably looks like a lot, but unfortunately it is probably not
# even a third of all such parameters that were necessary to accomplish
# two applications behind load balancers and with access to a few
# backend services such as MongoDB, redis, etcd and AMQP.
#
# Now, an improvement is to have a script (this) group hosts in
# a Cosmos environment based on hostnames, and provide name-to-address
# mappings in either Hiera or in a fact.
#
# For eduID, this can be implemented rather trivially since the naming
# standard is pretty much function-site-number.eduid.se. For nunoc-ops
# it is, with a couple of exceptions, functionnumber.domain.
#
# So, having automatic mappings of groups-of-hosts to addresses would
# allow lots of improvements, but one level of indirection is missing -
# services moves between hosts. eduID currently runs Redis and etcd on
# the same hosts that run MongoDB, but it would not be ideal to bet on
# that being true forever. Even better is to define roles that maps to
# the automatically deduced groups, which in turn maps to hosts that
# maps to addresses.
#
# As a starting point, this script is used with a roles-in.yaml file
# containing just some roles-to-groups maps:
#
# ---
# cosmos:
#
# roles:
# mongodb_server:
# groups: [userdb]
# etcd_server:
# groups: [userdb]
# redis_server:
# groups: [userdb]
# app_server:
# groups: [signup, dashboard]
# frontend_server:
# groups: [fe]
#
#
# When this script runs (like this:
#
# ./scripts/cosmos-facts --dirs .eduid.se \
# --roles metadata/roles-in.yaml \
# --outfile global/overlay/etc/puppet/static-cosmos-facts.yaml)
#
# it will scan for all directories ending with '.eduid.se' in the Cosmos repository,
# resolve the hosts using DNS and update the data with hosts and groups entrys like this
# (data not in DNS _could_ be provided in the YAML file above, under cosmos['hosts']):
#
# ---
# cosmos:
# hosts:
# userdb-fre-1.eduid.se:
# addrs: [130.242.130.220, '2001:6b0:54:c3:5054:ff:fea0:1dc']
# userdb-tug-1.eduid.se:
# addrs: [130.242.130.221, '2001:6b0:54:c3:5054:ff:fea0:1dd']
# ...
# groups:
# userdb:
# addrs: [130.242.130.220, 130.242.130.221, '2001:6b0:54:c3:5054:ff:fea0:1dc',
# '2001:6b0:54:c3:5054:ff:fea0:1dd']
# hosts: [userdb-fre-1.eduid.se, userdb-tug-1.eduid.se]
# ...
#
# The paramount use cases for this information in all our Puppet manifests
# is to do something (update firewall rules, write configuration files) with lists
# of IP addresses or hostnames. While making (sorted) lists of hostnames and addresses
# under the 'roles' section would be beautiful from a programmers perspective:
#
# $facts['cosmos']['roles']['mongodb_server']['hosts'] = ['userdb-fre-1.eduid.se', ...]
#
# it feels like it would be rather tedious to write and maintain such Puppet manifests.
# As a compromise, the data is still structurally under the 'cosmos' key but made
# available in the more palatable form:
#
# $facts['cosmos']['mongodb_server_addrs'] = ['130.242.130.220', '130.242.130.221',
# '2001:6b0:54:c3:5054:ff:fea0:1dc', '2001:6b0:54:c3:5054:ff:fea0:1dd']
#
# $facts['cosmos']['mongodb_server_hosts'] = ['userdb-fre-1.eduid.se', 'userdb-tug-1.eduid.se']
#
# To really encourage the use of roles rather than hosts/groups directly, the
# top-level 'cosmos' keys 'hosts', 'groups' and 'roles' are emptied before this script
# writes it's output unless the --output_hosts, --output_groups and --output_roles
# arguments are provided. Please don't use them ;).
#
import re
import sys
import copy
import yaml
import cosmosdata
from cosmosdata import DNSResolver, HostsResolver
DEFAULT_REGEXPS = [re.compile('(.+?)-'), # eduID style, function-site-digit
re.compile('([a-z]+)\d'), # nunoc-ops style, functiondigit
]
class Hosts(object):
def __init__(self, data_in, args):
self._data = {} if data_in is None else data_in
self._data.update({'hosts': {},
'groups': {},
})
self._args = args
if args.roles:
with open(args.roles) as fd:
file_y = yaml.safe_load(fd)
self._data.update(file_y.get('cosmos'))
@property
def hosts(self):
return self._data['hosts']
@property
def groups(self):
return self._data['groups']
def add_host(self, hostname, addrs):
if hostname not in self.hosts:
self.hosts[hostname] = {}
prev_addrs = self.hosts[hostname].get('addrs', [])
self.hosts[hostname]['addrs'] = _dedup(addrs + prev_addrs)
def add_group(self, groupname, data):
# rename 'members' to 'hosts'
data['hosts'] = data.pop('members')
self.groups.update({groupname: data})
# Make a list of all the addresses of the hosts
if 'addrs' not in self.groups[groupname]:
self.groups[groupname]['addrs'] = []
for this in self.groups[groupname]['hosts']:
addrs = self.groups[groupname]['addrs']
if this in self.hosts:
addrs += self.hosts[this]['addrs']
self.groups[groupname]['addrs'] = _dedup(addrs)
def load_datasource(self, data_s):
"""
:type data_s: cosmosdata.DataSource
:return:
"""
# Load all the hosts
if '_regexps' in self.groups:
group_r = [re.compile(x) for x in self.groups.pop('_regexps')]
else:
group_r = [re.compile(x) for x in DEFAULT_REGEXPS]
for this in data_s.all_hosts():
this = this.lower()
addrs = data_s.lookup(this)
if not addrs:
continue
self.add_host(this, addrs)
for r in group_r:
match = r.match(this)
if match:
data_s.add_members_to_group(match.group(1), [this])
# Add all the groups from the datasource
for this, values in data_s.groups().items():
self.add_group(this, values)
def to_dict(self, ignore_nonex):
res = copy.deepcopy(self._data)
if self._args.addfile:
# Add content from a specified file
fd = open(self._args.addfile)
file_y = yaml.safe_load(fd)
res.update(file_y)
res = _resolve_roles(res, ignore_nonex)
return dict(cosmos = res)
def _dedup(data):
t = {}
for x in data:
t[x] = 1
return sorted(t.keys())
def _resolve_roles(data, ignore_nonex):
for groupname, values in data.get('roles').items():
# Roles look like this:
# monitor_server:
# groups: [nagios]
# hosts: [monitor.sunet.se]
#
groupname = groupname.lower()
if 'groups' not in values:
continue
# merge 'hosts' and 'addrs' from all the referred groups
addrs = []
hosts = []
for g in values.get('groups', []):
if g not in data['groups']:
if ignore_nonex:
continue
raise ValueError('While resolving role {!r}, could not find group {!r} in groups: {}'.format(
groupname, g, list(data['groups'].keys())))
addrs = addrs + data['groups'][g]['addrs']
hosts = hosts + data['groups'][g]['hosts']
for h in values.get('hosts', []):
if h not in data['hosts']:
if ignore_nonex:
continue
raise ValueError('While resolving role {!r}, could not find host {!r}'.format(
groupname, h))
addrs = addrs + data['hosts'][h]['addrs']
hosts = hosts + [h]
# While structured data is nice, Puppet code like
# $facts['cosmos']['roles']['frontend_servers']['hosts']
# would be pretty tedious to maintain. Turn the output of
# role resolving (which is what is expected would be used most) into
# $facts['cosmos']['frontend_server_hosts']
# $facts['cosmos']['frontend_server_addrs']
# instead.
data[groupname + '_hosts'] = _dedup(hosts)
data[groupname + '_addrs'] = _dedup(addrs)
return data
def _make_header(data, args):
""" Make a header with the three hyphens indicating YAML, and some comments. """
roles = {}
groups = {}
# Make hash of roles -> groups and groups -> roles
for role, values in data['roles'].items():
for g in values.get('groups', []):
if role not in roles:
roles[role] = []
roles[role] += [g]
if g not in groups:
groups[g] = []
groups[g] += [role]
res = ['#',
'# This file was created by {}.'.format(sys.argv[0]),
'#',
'# {} roles defined in {}'.format(len(data['roles']), args.roles),
'# {} groups formed in the current Cosmos environment'.format(len(data['groups'])),
'#',
'# Roles <- groups:',
]
for role, _groups in sorted(roles.items()):
res += ['# {:20s} <- {}'.format(role, ', '.join(_dedup(_groups)))]
res += ['#',
'# Groups -> roles:',
]
for group, _roles in sorted(groups.items()):
res += ['# {:20s} -> {}'.format(group, ', '.join(_dedup(_roles)))]
unused = []
for g in data['groups'].keys():
if g not in groups:
unused += [g]
if unused:
res += ['#',
'# Unused groups: {}'.format(', '.join(sorted(unused))),
]
res += ['#',
'---',
'']
return '\n'.join(res)
def parse_args():
"""
Parse the command line arguments
"""
parser = cosmosdata.get_parser('Tool to create the "cosmos" fact',
['debug', 'dirs', 'outfile', 'hostsfile', 'addfile',
'classesfile'],
{})
parser.add_argument('--roles',
dest='roles',
default=None,
help='Input YAML file',
metavar='FILENAME',
)
for attr in ['hosts', 'groups', 'roles']:
parser.add_argument('--output_{}'.format(attr),
dest='output_{}'.format(attr),
action='store_true',
help='Output the "{}" section'.format(attr),
)
parser.add_argument('--ignore_nonexistent',
dest='ignore_nonex',
action='store_true',
help='Ignore non-existent data',
)
return parser.parse_args()
def main(args=None, data_in=None):
if args is None:
args = parse_args()
hosts = Hosts(data_in, args)
if args.dirs:
data_s = DNSResolver(args.dirs)
elif args.hostsfile:
data_s = HostsResolver(args.hostsfile)
else:
raise SyntaxError('Neither dirs nor hostsfile supplied')
if args.classesfile:
data_s.load_classes(args.classesfile)
hosts.load_datasource(data_s)
res = hosts.to_dict(ignore_nonex=args.ignore_nonex)
header = _make_header(res['cosmos'], args)
# To encourage use of roles, delete all this other data from the
# output unless specifically asked to keep them
for attr in ['hosts', 'groups', 'roles']:
if not getattr(args, 'output_' + attr) and attr in res['cosmos']:
del(res['cosmos'][attr])
else:
print('Keeping {}'.format(attr))
if args.outfile:
with open(args.outfile, 'w') as fd:
fd.write(header)
yaml.safe_dump(res, fd)
else:
print(header)
print(yaml.safe_dump(res))
return True
if __name__=='__main__':
try:
if main():
sys.exit(0)
sys.exit(1)
except KeyboardInterrupt:
pass

View file

@ -0,0 +1,2 @@
from cosmosdata.argv import get_parser
from cosmosdata.datasource import DataSource, DNSResolver, HostsResolver

View file

@ -0,0 +1,69 @@
import argparse
DEFAULTS = {'debug': False,
}
def get_parser(description, arguments, defaults):
"""
:param description: Program description
:param arguments: List of well known arguments in use
:param defaults: Default values
:return: ArgumentParser
:rtype: argparse.ArgumentParser
"""
parser = argparse.ArgumentParser(description = description,
add_help = True,
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
)
for this in arguments:
if this == 'debug':
parser.add_argument('--debug',
dest='debug',
action='store_true', default=defaults.get(this, DEFAULTS.get(this)),
help='Enable debug operation'
)
elif this == 'dirs':
parser.add_argument('--dirs',
dest = 'dirs',
default = None,
nargs = '*',
help = 'Find hosts based on cosmos host directorys ending with SUFFIX',
metavar = 'SUFFIX',
)
elif this == 'outfile':
parser.add_argument('--outfile',
dest = 'outfile',
default = None,
help = 'Output filename (to not print to stdout)',
metavar = 'FILENAME',
)
elif this == 'hostsfile':
parser.add_argument('--hostsfile',
dest = 'hostsfile',
default = None,
help = 'Input hosts YAML file',
metavar = 'FILENAME',
)
elif this == 'addfile':
parser.add_argument('--addfile',
dest = 'addfile',
default = None,
help = 'Add contents of this YAML file to the output',
metavar = 'FILENAME',
)
elif this == 'classesfile':
parser.add_argument('--classesfile',
dest='classesfile',
default=None,
help='Input classes YAML file (cosmos-rules.yaml)',
metavar='FILENAME',
)
else:
raise ValueError('Unknown argument: {}'.format(this))
return parser

View file

@ -0,0 +1,181 @@
import os
import re
import sys
import yaml
import socket
class DataSource(object):
def __init__(self):
self._groups = {}
def lookup(self, hostname):
raise NotImplemented('subclass must implement lookup')
def all_hosts(self):
raise NotImplemented('subclass must implement all_hosts')
def groups(self):
"""
Return {group: {members: [hosts]}} data that is used in creating hostgroups
"""
return self._groups
def add_members_to_group(self, group, hosts):
"""
Add hosts (FQDNs) to a group.
:type group: str
:type hosts: [str]
"""
if not hosts:
return
if group not in self._groups:
self._groups[group] = {'members': []}
for this in hosts:
if this not in self._groups[group]['members']:
self._groups[group]['members'] += [this]
def load_classes(self, filename):
with open(filename) as fd:
_classes = yaml.safe_load(fd)
for this_r, classes in _classes.items():
r_comp = re.compile(this_r)
hosts = []
for host in self.all_hosts():
if re.match(r_comp, host):
hosts += [host]
if not classes:
continue
for cls in classes.keys():
parts = cls.split('::')
if parts[0] in ['sunet', 'eduid', 'thulin']:
parts = parts[1:]
cls = '_'.join(parts)
self.add_members_to_group(cls, hosts)
class DNSResolver(DataSource):
"""
DNS based hostname-to-IPs resolver, returning both IPv4 and IPv6 addresses.
"""
def __init__(self, suffixes):
super(DNSResolver, self).__init__()
self._suffixes = suffixes
def lookup(self, hostname):
res = gai4 = gai6 = []
try:
gai4 = socket.getaddrinfo(hostname, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
except socket.gaierror:
sys.stderr.write('Host {} not found in DNS (v4)\n'.format(hostname))
try:
gai6 = socket.getaddrinfo(hostname, 0, socket.AF_INET6, 0, socket.IPPROTO_TCP)
except socket.gaierror:
sys.stderr.write('Host {} not found in DNS (v6)\n'.format(hostname))
for this in gai4 + gai6:
res.append(this[4][0])
return res
def all_hosts(self):
"""
Return a list of all hosts in this cosmos environment.
Currently this is done by looking for subdirectorys with a '.eduid.se' in them,
but not starting with a '.'.
"""
hosts = filter(lambda fn: os.path.isdir(fn) and \
not fn.startswith('.'),
os.listdir('.'))
matching = []
for h in hosts:
for s in self._suffixes:
if h.endswith(s):
matching += [h]
break
return sorted(matching)
class HostsResolver(DataSource):
"""
Return data from a YAML file with entrys like this:
site_hosts:
host1_vlan10:
addrs:
- 192.168.10.1
dhcp: false
fqdn: host1.example.org
mac: xxx
host2_vlan2:
addrs:
- 192.168.2.2
- 2001::1
- 2001::2
dhcp: true
fqdn: host2.example.org
mac: yyy
type: server
"""
def __init__(self, fn='global/overlay/etc/puppet/hiera/data/hosts.yaml'):
super(HostsResolver, self).__init__()
self._fn = fn
self._data = _load_hosts_file(fn)
def lookup(self, hostname):
hostname = hostname.lower()
res = []
for this, values in self._data.items():
fqdn = self._get_fqdn(this)
if not fqdn:
continue
if fqdn == hostname:
res += values.get('addrs', [])
return res
def all_hosts(self):
"""
Return a list of all FQDNs in this cosmos environment.
"""
res = {}
for this in self._data.keys():
fqdn = self._get_fqdn(this)
if not fqdn:
continue
res[fqdn] = True
return res.keys()
def groups(self):
"""
Create hostgroups based on 'type' in the hosts data.
"""
res = super(HostsResolver, self).groups()
for this, values in self._data.items():
if 'type' in values:
group = values['type'].lower()
fqdn = self._get_fqdn(this)
if fqdn is None:
continue
self.add_members_to_group(group, [fqdn])
return res
def _get_fqdn(self, key):
fqdn = self._data[key]['fqdn'].lower()
if '-old.' in fqdn:
return None
if self._data[key].get('monitor') is False:
return None
return fqdn
def _load_hosts_file(fn):
"""
Load site-hosts data from hosts.yaml.
"""
with open(fn) as fd:
data = yaml.load(fd)
return data['site_hosts']

View file

@ -0,0 +1,141 @@
import os
import sys
import time
import yaml
import subprocess
def run(cmd, input=None):
"""
Run script, storing various aspects of the results.
"""
data = {}
data['start_time'] = time.time()
proc = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True,
)
#print("Run command {!r} with input {}".format(cmd, input))
(stdout, stderr) = proc.communicate(input=input)
data['end_time'] = time.time()
data['exit_status'] = proc.returncode
data['pid'] = proc.pid
data['output'] = stdout
data['stderr'] = stderr
data['success'] = True
if stderr is not None or proc.returncode:
sys.stderr.write('Command {!r} FAILED (exit {}), stdout {} stderr: {}\n'.format(
' '.join(cmd), proc.returncode, stdout, stderr))
data['success'] = False
return data
def extract_comments(data):
comments = {}
curr = []
for this in data.split('\n'):
if this.startswith('#'):
#print("COMMENT: {}".format(this))
curr += [this]
elif ':' in this and len(curr):
key = this.split(':')[0]
comments[key] = curr
#print("KEY {} FROM {}: {}".format(key, this, curr))
curr = []
else:
#print("RESET AT {}".format(this))
curr = []
return comments
def get_host_pubkey_fn(args, fqdn):
fns = [os.path.join(args.certdir, '{}_infra.pem'.format(fqdn)),
os.path.join(args.certdir, '{}.pem'.format(fqdn)),
os.path.join(args.certdir, 'public_certkey.pkcs7.pem'),
]
for fn in fns:
if os.path.isfile(fn):
return fn
print('Public key file not found (tried {})'.format(pubkey_fn, fns))
return None
def get_host_eyaml_fn(fqdn):
return '{}/overlay/etc/hiera/data/local.eyaml'.format(fqdn)
def get_host_secrets_fn(fqdn):
return '{}/overlay/etc/hiera/data/secrets.yaml.asc'.format(fqdn)
def get_host_secrets(fqdn):
"""
SSH to a host and read all it's secrets.
:param fqdn: Fully qualified hostname
:return: Two dicts (data, comments)
"""
data = run(['ssh',
fqdn,
'GNUPGHOME=/etc/hiera/gpg/ gpg -d /etc/hiera/data/secrets.yaml.asc 2>/dev/null'])
if not data['success']:
return False, False
return _parse_eyaml(data['output'].decode('utf-8'))
def load_host_eyaml(fn):
"""
Load an eyaml file and return parsed data plus comments
:param fn: EYAML filename
:return: Two dicts (data, comments)
"""
if not os.path.isfile(fn):
return {}, {}
with open(fn) as fd:
data_str = fd.read(1024 * 1024)
return _parse_eyaml(data_str)
def _parse_eyaml(data):
"""
Parse YAML data but also retain comments associated with the keys.
:param data:
:return: Two dicts (data, comments)
"""
if not data:
return False, False
try:
comments = extract_comments(data)
return yaml.safe_load(data), comments
except Exception as exc:
sys.stderr.write('Failed to parse EYAML:\n{}\n{}\n'.format(data, exc))
return False, False
def make_eyaml(data, comments, pubkey_fn):
res = []
for k, v in sorted(data.items()):
cmd = ['eyaml',
'encrypt',
'--label', k,
'--output', 'block',
'--stdin',
'--pkcs7-public-key', pubkey_fn,
]
eyaml = run(cmd, input=bytes(str(v), 'utf-8'))
if not eyaml['success']:
return False
if k in comments:
res += comments[k]
enc = eyaml['output'].decode('utf-8')
if 'PKCS' not in enc:
print("Command '{}' failed: {}".format(' '.join(cmd), enc))
res += enc.split('\n')
return res