added facts

global/overlay/etc/puppet/static-cosmos-facts.yaml

@@ -0,0 +1,26 @@
+#
+# This file was created by ./scripts/cosmos-facts.
+#
+# 1 roles defined in metadata/roles-in.yaml
+# 54 groups formed in the current Cosmos environment
+#
+# Roles <- groups:
+#    frontend_server      <- fe
+#
+# Groups -> roles:
+#    fe                   -> frontend_server
+#
+# Unused groups: auditd, autoupdate, common, country, dockerhost2, eid_de_client, eid_dockerhost, eid_idm_app, eid_idm_db, eid_idm_redis, eid_kvmhost, eid_nagios_monitor, eidas_hsm_client, eidas_metadata_key, entropyclient, eumd, eupub, fe_servers, frontend_load_balancer, frontend_register_sites, idm, infra_ca_rp, invent_client, jump, jumphosts, konsulter, kvmdemw, kvmeidas, kvmfe, kvminfra, kvmmeta, mailclient, md, md_publisher, md_repo_client, md_repo_server, md_signer, mdsl_publisher, metadata_mdq_publisher, metadata_mdqp, metadata_metadata_repo, metadata_pyff_compose, monitor, natmd, natpub, nrpe, p, r, rsyslog, server, sunet_iaas_cloud, sunetops, validator
+#
+---
+cosmos:
+  frontend_server_addrs:
+  - 94.176.224.165
+  - 94.176.224.166
+  - 94.176.224.37
+  - 94.176.224.38
+  frontend_server_hosts:
+  - fe-fre-1.test.komreg.net
+  - fe-fre-3.komreg.net
+  - fe-tug-1.test.komreg.net
+  - fe-tug-3.komreg.net

metadata/roles-in.yaml

@@ -0,0 +1,6 @@
+---
+cosmos:
+
+  roles:
+    frontend_server:
+      groups: [fe]

scripts/cosmos-facts

@@ -0,0 +1,441 @@
+#!/usr/bin/env python3
+#
+# Copyright 2017 SUNET. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+#    1. Redistributions of source code must retain the above copyright notice, this list of
+#       conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright notice, this list
+#       of conditions and the following disclaimer in the documentation and/or other materials
+#       provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY SUNET ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SUNET OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and documentation are those of the
+# authors and should not be interpreted as representing official policies, either expressed
+# or implied, of SUNET.
+#
+# Author : Fredrik Thulin <fredrik@thulin.net>
+#
+
+#
+# TL; DR: This script produces a Puppet fact called 'cosmos', with information
+# about hosts in the current Cosmos environment:
+#
+#   $facts['cosmos']['mongodb_server_addrs'] = [130.242.130.220, 130.242.130.221,
+#     '2001:6b0:54:c3:5054:ff:fea0:1dc', '2001:6b0:54:c3:5054:ff:fea0:1dd']
+#
+#   $facts['cosmos']['mongodb_server_hosts'] = [userdb-fre-1.eduid.se, userdb-tug-1.eduid.se]
+#
+
+#
+# This script creates the $facts['cosmos'] Puppet fact. This fact holds
+# information about the Cosmos environment. To start with, this fact is
+# used to define roles of hosts in the Cosmos environment.
+#
+# An example to provide the merits for this fact, and how it works:
+#
+#   eduID has the two applications 'signup' and 'dashboard'. They are
+#   quite common and both have access to MongoDB, both register with
+#   frontends etc.
+#
+#   eduID had lots of Hiera data like this: (different values in test
+#   and production, making it harder than necessary to deploy things
+#   from testing to production):
+#
+#     eduid_frontend_servers:
+#       - fe-tug-1.eduid.se
+#       - fe-fre-1.eduid.se
+#
+#     eduid_frontend_ips:
+#       - 130.242.131.3
+#       - 130.242.131.4
+#       - 2001:6b0:54:c4::3
+#       - 2001:6b0:54:c4::4
+#
+#     eduid_mongodb_servers:
+#       - userdb-tug-1.eduid.se
+#       - userdb-fre-1.eduid.se
+#       - userdb-tug-2.eduid.se
+#
+#     eduid_signup_haproxy_backends:
+#       - signup-tug-1.eduid.se
+#       #- signup-fre-1.eduid.se
+#
+#     eduid_dashboard_haproxy_backends:
+#       - dash-tug-1.eduid.se
+#       - dash-fre-1.eduid.se
+#
+#     eduid_signup_ips:
+#       - 130.242.130.212                     # signup-tug-1.eduid.se
+#       - 2001:6b0:54:c3:5054:ff:fea0:1d4     # signup-tug-1.eduid.se
+#     eduid_dashboard_ips:
+#       - 130.242.130.213                     # dash-fre-1.eduid.se
+#       - 130.242.130.197                     # dash-tug-1.eduid.se
+#       - 2001:6b0:54:c3:5054:ff:fea0:1d5     # dash-fre-1.eduid.se
+#       - 2001:6b0:54:c3:5054:ff:fea0:1c5     # dash-tug-1.eduid.se
+#
+# This probably looks like a lot, but unfortunately it is probably not
+# even a third of all such parameters that were necessary to accomplish
+# two applications behind load balancers and with access to a few
+# backend services such as MongoDB, redis, etcd and AMQP.
+#
+# Now, an improvement is to have a script (this) group hosts in
+# a Cosmos environment based on hostnames, and provide name-to-address
+# mappings in either Hiera or in a fact.
+#
+# For eduID, this can be implemented rather trivially since the naming
+# standard is pretty much function-site-number.eduid.se. For nunoc-ops
+# it is, with a couple of exceptions, functionnumber.domain.
+#
+# So, having automatic mappings of groups-of-hosts to addresses would
+# allow lots of improvements, but one level of indirection is missing -
+# services moves between hosts. eduID currently runs Redis and etcd on
+# the same hosts that run MongoDB, but it would not be ideal to bet on
+# that being true forever. Even better is to define roles that maps to
+# the automatically deduced groups, which in turn maps to hosts that
+# maps to addresses.
+#
+# As a starting point, this script is used with a roles-in.yaml file
+# containing just some roles-to-groups maps:
+#
+#   ---
+#   cosmos:
+#
+#     roles:
+#       mongodb_server:
+#         groups: [userdb]
+#       etcd_server:
+#         groups: [userdb]
+#       redis_server:
+#         groups: [userdb]
+#       app_server:
+#         groups: [signup, dashboard]
+#      frontend_server:
+#         groups: [fe]
+#
+#
+# When this script runs (like this:
+#
+#    ./scripts/cosmos-facts --dirs .eduid.se \
+#        --roles metadata/roles-in.yaml \
+#        --outfile global/overlay/etc/puppet/static-cosmos-facts.yaml)
+#
+# it will scan for all directories ending with '.eduid.se' in the Cosmos repository,
+# resolve the hosts using DNS and update the data with hosts and groups entrys like this
+# (data not in DNS _could_ be provided in the YAML file above, under cosmos['hosts']):
+#
+# ---
+# cosmos:
+#   hosts:
+#     userdb-fre-1.eduid.se:
+#       addrs: [130.242.130.220, '2001:6b0:54:c3:5054:ff:fea0:1dc']
+#     userdb-tug-1.eduid.se:
+#       addrs: [130.242.130.221, '2001:6b0:54:c3:5054:ff:fea0:1dd']
+#     ...
+#   groups:
+#     userdb:
+#       addrs: [130.242.130.220, 130.242.130.221, '2001:6b0:54:c3:5054:ff:fea0:1dc',
+#         '2001:6b0:54:c3:5054:ff:fea0:1dd']
+#       hosts: [userdb-fre-1.eduid.se, userdb-tug-1.eduid.se]
+#     ...
+#
+# The paramount use cases for this information in all our Puppet manifests
+# is to do something (update firewall rules, write configuration files) with lists
+# of IP addresses or hostnames. While making (sorted) lists of hostnames and addresses
+# under the 'roles' section would be beautiful from a programmers perspective:
+#
+#   $facts['cosmos']['roles']['mongodb_server']['hosts'] = ['userdb-fre-1.eduid.se', ...]
+#
+# it feels like it would be rather tedious to write and maintain such Puppet manifests.
+# As a compromise, the data is still structurally under the 'cosmos' key but made
+# available in the more palatable form:
+#
+#   $facts['cosmos']['mongodb_server_addrs'] = ['130.242.130.220', '130.242.130.221',
+#     '2001:6b0:54:c3:5054:ff:fea0:1dc', '2001:6b0:54:c3:5054:ff:fea0:1dd']
+#
+#   $facts['cosmos']['mongodb_server_hosts'] = ['userdb-fre-1.eduid.se', 'userdb-tug-1.eduid.se']
+#
+# To really encourage the use of roles rather than hosts/groups directly, the
+# top-level 'cosmos' keys 'hosts', 'groups' and 'roles' are emptied before this script
+# writes it's output unless the --output_hosts, --output_groups and --output_roles
+# arguments are provided. Please don't use them ;).
+#
+
+
+import re
+import sys
+import copy
+import yaml
+
+
+import cosmosdata
+from cosmosdata import DNSResolver, HostsResolver
+
+
+DEFAULT_REGEXPS = [re.compile('(.+?)-'),      # eduID style, function-site-digit
+                   re.compile('([a-z]+)\d'),  # nunoc-ops style, functiondigit
+                   ]
+
+
+class Hosts(object):
+
+    def __init__(self, data_in, args):
+        self._data = {} if data_in is None else data_in
+
+        self._data.update({'hosts': {},
+                           'groups': {},
+                           })
+
+        self._args = args
+
+        if args.roles:
+            with open(args.roles) as fd:
+                file_y = yaml.safe_load(fd)
+                self._data.update(file_y.get('cosmos'))
+
+    @property
+    def hosts(self):
+        return self._data['hosts']
+
+    @property
+    def groups(self):
+        return self._data['groups']
+
+    def add_host(self, hostname, addrs):
+        if hostname not in self.hosts:
+            self.hosts[hostname] = {}
+        prev_addrs = self.hosts[hostname].get('addrs', [])
+        self.hosts[hostname]['addrs'] = _dedup(addrs + prev_addrs)
+
+    def add_group(self, groupname, data):
+        # rename 'members' to 'hosts'
+        data['hosts'] = data.pop('members')
+        self.groups.update({groupname: data})
+        # Make a list of all the addresses of the hosts
+        if 'addrs' not in self.groups[groupname]:
+            self.groups[groupname]['addrs'] = []
+        for this in self.groups[groupname]['hosts']:
+            addrs = self.groups[groupname]['addrs']
+            if this in self.hosts:
+                addrs += self.hosts[this]['addrs']
+            self.groups[groupname]['addrs'] = _dedup(addrs)
+
+    def load_datasource(self, data_s):
+        """
+        :type data_s: cosmosdata.DataSource
+        :return:
+        """
+        # Load all the hosts
+        if '_regexps' in self.groups:
+            group_r = [re.compile(x) for x in self.groups.pop('_regexps')]
+        else:
+            group_r =  [re.compile(x) for x in DEFAULT_REGEXPS]
+        for this in data_s.all_hosts():
+            this = this.lower()
+            addrs = data_s.lookup(this)
+            if not addrs:
+                continue
+            self.add_host(this, addrs)
+            for r in group_r:
+                match = r.match(this)
+                if match:
+                    data_s.add_members_to_group(match.group(1), [this])
+        # Add all the groups from the datasource
+        for this, values in data_s.groups().items():
+            self.add_group(this, values)
+
+    def to_dict(self, ignore_nonex):
+        res = copy.deepcopy(self._data)
+        if self._args.addfile:
+            # Add content from a specified file
+            fd = open(self._args.addfile)
+            file_y = yaml.safe_load(fd)
+            res.update(file_y)
+        res = _resolve_roles(res, ignore_nonex)
+        return dict(cosmos = res)
+
+
+def _dedup(data):
+    t = {}
+    for x in data:
+        t[x] = 1
+    return sorted(t.keys())
+
+def _resolve_roles(data, ignore_nonex):
+    for groupname, values in data.get('roles').items():
+        # Roles look like this:
+        #    monitor_server:
+        #      groups: [nagios]
+        #      hosts: [monitor.sunet.se]
+        #
+        groupname = groupname.lower()
+        if 'groups' not in values:
+            continue
+        # merge 'hosts' and 'addrs' from all the referred groups
+        addrs = []
+        hosts = []
+        for g in values.get('groups', []):
+            if g not in data['groups']:
+                if ignore_nonex:
+                    continue
+                raise ValueError('While resolving role {!r}, could not find group {!r} in groups: {}'.format(
+                    groupname, g, list(data['groups'].keys())))
+            addrs = addrs + data['groups'][g]['addrs']
+            hosts = hosts + data['groups'][g]['hosts']
+        for h in values.get('hosts', []):
+            if h not in data['hosts']:
+                if ignore_nonex:
+                    continue
+                raise ValueError('While resolving role {!r}, could not find host {!r}'.format(
+                    groupname, h))
+            addrs = addrs + data['hosts'][h]['addrs']
+            hosts = hosts + [h]
+        # While structured data is nice, Puppet code like
+        #   $facts['cosmos']['roles']['frontend_servers']['hosts']
+        # would be pretty tedious to maintain. Turn the output of
+        # role resolving (which is what is expected would be used most) into
+        #   $facts['cosmos']['frontend_server_hosts']
+        #   $facts['cosmos']['frontend_server_addrs']
+        # instead.
+        data[groupname + '_hosts'] = _dedup(hosts)
+        data[groupname + '_addrs'] = _dedup(addrs)
+
+    return data
+
+def _make_header(data, args):
+    """ Make a header with the three hyphens indicating YAML, and some comments. """
+    roles = {}
+    groups = {}
+
+    # Make hash of roles -> groups and groups -> roles
+    for role, values in data['roles'].items():
+        for g in values.get('groups', []):
+            if role not in roles:
+                roles[role] = []
+            roles[role] += [g]
+
+            if g not in groups:
+                groups[g] = []
+            groups[g] += [role]
+
+    res = ['#',
+           '# This file was created by {}.'.format(sys.argv[0]),
+           '#',
+           '# {} roles defined in {}'.format(len(data['roles']), args.roles),
+           '# {} groups formed in the current Cosmos environment'.format(len(data['groups'])),
+           '#',
+           '# Roles <- groups:',
+           ]
+    for role, _groups in sorted(roles.items()):
+        res += ['#    {:20s} <- {}'.format(role, ', '.join(_dedup(_groups)))]
+    res += ['#',
+            '# Groups -> roles:',
+            ]
+    for group, _roles in sorted(groups.items()):
+        res += ['#    {:20s} -> {}'.format(group, ', '.join(_dedup(_roles)))]
+
+    unused = []
+    for g in data['groups'].keys():
+        if g not in groups:
+            unused += [g]
+    if unused:
+        res += ['#',
+                '# Unused groups: {}'.format(', '.join(sorted(unused))),
+                ]
+
+
+    res += ['#',
+            '---',
+            '']
+    return '\n'.join(res)
+
+def parse_args():
+    """
+    Parse the command line arguments
+    """
+    parser = cosmosdata.get_parser('Tool to create the "cosmos" fact',
+                                  ['debug', 'dirs', 'outfile', 'hostsfile', 'addfile',
+                                   'classesfile'],
+                                  {})
+    parser.add_argument('--roles',
+                        dest='roles',
+                        default=None,
+                        help='Input YAML file',
+                        metavar='FILENAME',
+                        )
+
+    for attr in ['hosts', 'groups', 'roles']:
+        parser.add_argument('--output_{}'.format(attr),
+                            dest='output_{}'.format(attr),
+                            action='store_true',
+                            help='Output the "{}" section'.format(attr),
+                            )
+    parser.add_argument('--ignore_nonexistent',
+                        dest='ignore_nonex',
+                        action='store_true',
+                        help='Ignore non-existent data',
+                        )
+
+
+    return parser.parse_args()
+
+
+def main(args=None, data_in=None):
+    if args is None:
+        args = parse_args()
+
+    hosts = Hosts(data_in, args)
+    if args.dirs:
+        data_s = DNSResolver(args.dirs)
+    elif args.hostsfile:
+        data_s = HostsResolver(args.hostsfile)
+    else:
+        raise SyntaxError('Neither dirs nor hostsfile supplied')
+
+    if args.classesfile:
+        data_s.load_classes(args.classesfile)
+
+    hosts.load_datasource(data_s)
+
+    res = hosts.to_dict(ignore_nonex=args.ignore_nonex)
+
+    header = _make_header(res['cosmos'], args)
+
+    # To encourage use of roles, delete all this other data from the
+    # output unless specifically asked to keep them
+    for attr in ['hosts', 'groups', 'roles']:
+        if not getattr(args, 'output_' + attr) and attr in res['cosmos']:
+            del(res['cosmos'][attr])
+        else:
+            print('Keeping {}'.format(attr))
+
+    if args.outfile:
+        with open(args.outfile, 'w') as fd:
+            fd.write(header)
+            yaml.safe_dump(res, fd)
+    else:
+        print(header)
+        print(yaml.safe_dump(res))
+
+    return True
+
+
+if __name__=='__main__':
+    try:
+        if main():
+            sys.exit(0)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        pass

scripts/cosmosdata/__init__.py

@@ -0,0 +1,2 @@
+from cosmosdata.argv import get_parser
+from cosmosdata.datasource import DataSource, DNSResolver, HostsResolver

scripts/cosmosdata/argv.py

@@ -0,0 +1,69 @@
+import argparse
+
+
+DEFAULTS = {'debug': False,
+            }
+
+
+def get_parser(description, arguments, defaults):
+    """
+
+    :param description: Program description
+    :param arguments: List of well known arguments in use
+    :param defaults: Default values
+
+    :return: ArgumentParser
+    :rtype: argparse.ArgumentParser
+    """
+    parser = argparse.ArgumentParser(description = description,
+                                     add_help = True,
+                                     formatter_class = argparse.ArgumentDefaultsHelpFormatter,
+                                     )
+    for this in arguments:
+        if this == 'debug':
+            parser.add_argument('--debug',
+                                dest='debug',
+                                action='store_true', default=defaults.get(this, DEFAULTS.get(this)),
+                                help='Enable debug operation'
+                                )
+        elif this == 'dirs':
+            parser.add_argument('--dirs',
+                                dest = 'dirs',
+                                default = None,
+                                nargs = '*',
+                                help = 'Find hosts based on cosmos host directorys ending with SUFFIX',
+                                metavar = 'SUFFIX',
+                                )
+        elif this == 'outfile':
+            parser.add_argument('--outfile',
+                                dest = 'outfile',
+                                default = None,
+                                help = 'Output filename (to not print to stdout)',
+                                metavar = 'FILENAME',
+            )
+        elif this == 'hostsfile':
+            parser.add_argument('--hostsfile',
+                                dest = 'hostsfile',
+                                default = None,
+                                help = 'Input hosts YAML file',
+                                metavar = 'FILENAME',
+            )
+        elif this == 'addfile':
+            parser.add_argument('--addfile',
+                                dest = 'addfile',
+                                default = None,
+                                help = 'Add contents of this YAML file to the output',
+                                metavar = 'FILENAME',
+            )
+        elif this == 'classesfile':
+            parser.add_argument('--classesfile',
+                                dest='classesfile',
+                                default=None,
+                                help='Input classes YAML file (cosmos-rules.yaml)',
+                                metavar='FILENAME',
+                                )
+        else:
+            raise ValueError('Unknown argument: {}'.format(this))
+
+    return parser
+

scripts/cosmosdata/datasource.py

@@ -0,0 +1,181 @@
+import os
+import re
+import sys
+import yaml
+import socket
+
+
+class DataSource(object):
+
+    def __init__(self):
+        self._groups = {}
+
+    def lookup(self, hostname):
+        raise NotImplemented('subclass must implement lookup')
+
+    def all_hosts(self):
+        raise NotImplemented('subclass must implement all_hosts')
+
+    def groups(self):
+        """
+        Return {group: {members: [hosts]}} data that is used in creating hostgroups
+        """
+        return self._groups
+
+    def add_members_to_group(self, group, hosts):
+        """
+        Add hosts (FQDNs) to a group.
+        :type group: str
+        :type hosts: [str]
+        """
+        if not hosts:
+            return
+        if group not in self._groups:
+            self._groups[group] = {'members': []}
+        for this in hosts:
+            if this not in self._groups[group]['members']:
+                self._groups[group]['members'] += [this]
+
+    def load_classes(self, filename):
+        with open(filename) as fd:
+            _classes = yaml.safe_load(fd)
+            for this_r, classes in _classes.items():
+                r_comp = re.compile(this_r)
+                hosts = []
+                for host in self.all_hosts():
+                    if re.match(r_comp, host):
+                        hosts += [host]
+                if not classes:
+                    continue
+                for cls in classes.keys():
+                    parts = cls.split('::')
+                    if parts[0] in ['sunet', 'eduid', 'thulin']:
+                        parts = parts[1:]
+                    cls = '_'.join(parts)
+                    self.add_members_to_group(cls, hosts)
+
+
+class DNSResolver(DataSource):
+    """
+    DNS based hostname-to-IPs resolver, returning both IPv4 and IPv6 addresses.
+    """
+
+    def __init__(self, suffixes):
+        super(DNSResolver, self).__init__()
+        self._suffixes = suffixes
+
+    def lookup(self, hostname):
+        res = gai4 = gai6 = []
+        try:
+            gai4 = socket.getaddrinfo(hostname, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
+        except socket.gaierror:
+            sys.stderr.write('Host {} not found in DNS (v4)\n'.format(hostname))
+        try:
+            gai6 = socket.getaddrinfo(hostname, 0, socket.AF_INET6, 0, socket.IPPROTO_TCP)
+        except socket.gaierror:
+            sys.stderr.write('Host {} not found in DNS (v6)\n'.format(hostname))
+        for this in gai4 + gai6:
+            res.append(this[4][0])
+        return res
+
+    def all_hosts(self):
+        """
+        Return a list of all hosts in this cosmos environment.
+
+        Currently this is done by looking for subdirectorys with a '.eduid.se' in them,
+        but not starting with a '.'.
+        """
+        hosts = filter(lambda fn: os.path.isdir(fn) and \
+                       not fn.startswith('.'),
+
+                       os.listdir('.'))
+        matching = []
+        for h in hosts:
+            for s in self._suffixes:
+                if h.endswith(s):
+                    matching += [h]
+                    break
+        return sorted(matching)
+
+
+class HostsResolver(DataSource):
+    """
+    Return data from a YAML file with entrys like this:
+
+    site_hosts:
+       host1_vlan10:
+          addrs:
+          - 192.168.10.1
+          dhcp: false
+          fqdn: host1.example.org
+          mac: xxx
+       host2_vlan2:
+          addrs:
+            - 192.168.2.2
+            - 2001::1
+            - 2001::2
+          dhcp: true
+          fqdn: host2.example.org
+          mac: yyy
+          type: server
+    """
+
+    def __init__(self, fn='global/overlay/etc/puppet/hiera/data/hosts.yaml'):
+        super(HostsResolver, self).__init__()
+        self._fn = fn
+        self._data = _load_hosts_file(fn)
+
+    def lookup(self, hostname):
+        hostname = hostname.lower()
+        res = []
+        for this, values in self._data.items():
+            fqdn = self._get_fqdn(this)
+            if not fqdn:
+                continue
+            if fqdn == hostname:
+                res += values.get('addrs', [])
+        return res
+
+    def all_hosts(self):
+        """
+        Return a list of all FQDNs in this cosmos environment.
+        """
+        res = {}
+        for this in self._data.keys():
+            fqdn = self._get_fqdn(this)
+            if not fqdn:
+                continue
+            res[fqdn] = True
+        return res.keys()
+
+    def groups(self):
+        """
+        Create hostgroups based on 'type' in the hosts data.
+        """
+        res = super(HostsResolver, self).groups()
+        for this, values in self._data.items():
+            if 'type' in values:
+                group = values['type'].lower()
+                fqdn = self._get_fqdn(this)
+                if fqdn is None:
+                    continue
+                self.add_members_to_group(group, [fqdn])
+        return res
+
+    def _get_fqdn(self, key):
+        fqdn = self._data[key]['fqdn'].lower()
+        if '-old.' in fqdn:
+            return None
+        if self._data[key].get('monitor') is False:
+            return None
+        return fqdn
+
+
+def _load_hosts_file(fn):
+    """
+    Load site-hosts data from hosts.yaml.
+    """
+    with open(fn) as fd:
+        data = yaml.load(fd)
+    return data['site_hosts']
+

scripts/cosmosdata/secrets.py

@@ -0,0 +1,141 @@
+import os
+import sys
+import time
+import yaml
+import subprocess
+
+
+def run(cmd, input=None):
+    """
+    Run script, storing various aspects of the results.
+    """
+    data = {}
+    data['start_time'] = time.time()
+    proc = subprocess.Popen(cmd,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            close_fds=True,
+                            )
+    #print("Run command {!r} with input {}".format(cmd, input))
+    (stdout, stderr) = proc.communicate(input=input)
+    data['end_time'] = time.time()
+    data['exit_status'] = proc.returncode
+    data['pid'] = proc.pid
+    data['output'] = stdout
+    data['stderr'] = stderr
+    data['success'] = True
+    if stderr is not None or proc.returncode:
+        sys.stderr.write('Command {!r} FAILED (exit {}), stdout {} stderr: {}\n'.format(
+            ' '.join(cmd), proc.returncode, stdout, stderr))
+        data['success'] = False
+    return data
+
+def extract_comments(data):
+    comments = {}
+    curr = []
+    for this in data.split('\n'):
+        if this.startswith('#'):
+            #print("COMMENT: {}".format(this))
+            curr += [this]
+        elif ':' in this and len(curr):
+            key = this.split(':')[0]
+            comments[key] = curr
+            #print("KEY {} FROM {}: {}".format(key, this, curr))
+            curr = []
+        else:
+            #print("RESET AT {}".format(this))
+            curr = []
+    return comments
+
+
+def get_host_pubkey_fn(args, fqdn):
+    fns = [os.path.join(args.certdir, '{}_infra.pem'.format(fqdn)),
+           os.path.join(args.certdir, '{}.pem'.format(fqdn)),
+           os.path.join(args.certdir, 'public_certkey.pkcs7.pem'),
+           ]
+    for fn in fns:
+        if os.path.isfile(fn):
+            return fn
+    print('Public key file not found (tried {})'.format(pubkey_fn, fns))
+    return None
+
+
+def get_host_eyaml_fn(fqdn):
+    return '{}/overlay/etc/hiera/data/local.eyaml'.format(fqdn)
+
+
+def get_host_secrets_fn(fqdn):
+    return '{}/overlay/etc/hiera/data/secrets.yaml.asc'.format(fqdn)
+
+
+def get_host_secrets(fqdn):
+    """
+    SSH to a host and read all it's secrets.
+
+    :param fqdn: Fully qualified hostname
+    :return: Two dicts (data, comments)
+    """
+    data = run(['ssh',
+                fqdn,
+                'GNUPGHOME=/etc/hiera/gpg/ gpg -d /etc/hiera/data/secrets.yaml.asc 2>/dev/null'])
+
+    if not data['success']:
+        return False, False
+
+    return _parse_eyaml(data['output'].decode('utf-8'))
+
+
+def load_host_eyaml(fn):
+    """
+    Load an eyaml file and return parsed data plus comments
+
+    :param fn: EYAML filename
+    :return: Two dicts (data, comments)
+    """
+    if not os.path.isfile(fn):
+        return {}, {}
+
+    with open(fn) as fd:
+        data_str = fd.read(1024 * 1024)
+    return _parse_eyaml(data_str)
+
+
+def _parse_eyaml(data):
+    """
+    Parse YAML data but also retain comments associated with the keys.
+
+    :param data:
+    :return: Two dicts (data, comments)
+    """
+    if not data:
+        return False, False
+    try:
+        comments = extract_comments(data)
+        return yaml.safe_load(data), comments
+    except Exception as exc:
+        sys.stderr.write('Failed to parse EYAML:\n{}\n{}\n'.format(data, exc))
+
+    return False, False
+
+
+def make_eyaml(data, comments, pubkey_fn):
+    res = []
+    for k, v in sorted(data.items()):
+        cmd = ['eyaml',
+               'encrypt',
+               '--label', k,
+               '--output', 'block',
+               '--stdin',
+               '--pkcs7-public-key', pubkey_fn,
+        ]
+        eyaml = run(cmd, input=bytes(str(v), 'utf-8'))
+        if not eyaml['success']:
+            return False
+        if k in comments:
+            res += comments[k]
+        enc = eyaml['output'].decode('utf-8')
+        if 'PKCS' not in enc:
+            print("Command '{}' failed: {}".format(' '.join(cmd), enc))
+        res += enc.split('\n')
+    return res