diff --git a/Makefile b/Makefile index 8248ece..4c378d2 100644 --- a/Makefile +++ b/Makefile @@ -4,5 +4,9 @@ cosmos: upgrade: fab upgrade -bump: - ./bump-tag +db: + @python ./fabfile/db.py > global/overlay/etc/puppet/cosmos-db.yaml + @git add global/overlay/etc/puppet/cosmos-db.yaml && git commit -m "update db" global/overlay/etc/puppet/cosmos-db.yaml + +tag: db + ./bump-tag diff --git a/docs/cosmos-puppet-ops.mkd b/docs/cosmos-puppet-ops.mkd index 023c52c..46ceb50 100644 --- a/docs/cosmos-puppet-ops.mkd +++ b/docs/cosmos-puppet-ops.mkd @@ -210,6 +210,14 @@ as 'ro'. The read-only remote is used by multiverse scripts during host bootstra # git remote add ro git://yourhost/myproj-cosmos.git ``` +Now edit .git/config and rename the 'master' branch to use the new 'origin' remote or +you'll try to push to the multiverse remote! Finally create a branch for the 'multiverse' +upstream so you can merge changes to multiverse: + +``` +# git checkout -b multiverse --track multiverse/master +``` + Note that you can maintain your repo on just about any git hosting platform, including github, gitorius or your own local setup as long as it supports read-only "git://" access to your repository. It is important that the remotes called 'origin' and 'ro' refer to diff --git a/fabfile/__init__.py b/fabfile/__init__.py index caf123f..d87fbdd 100644 --- a/fabfile/__init__.py +++ b/fabfile/__init__.py @@ -1,44 +1,23 @@ from fabric.api import run,env -from fabric.operations import get +from fabric.operations import get,put import os import yaml import re - -def _all_hosts(): - return filter(lambda fn: '.' in fn and not fn.startswith('.') and os.path.isdir(fn),os.listdir(".")) - -def _roledefs(): - rules = dict() - - rules_file = "cosmos-rules.yaml"; - if os.path.exists(rules_file): - with open(rules_file) as fd: - rules.update(yaml.load(fd)) - - roles = dict() - for node_name in _all_hosts(): - for reg,cls in rules.iteritems(): - if re.search(reg,node_name): - for cls_name in cls.keys(): - h = roles.get(cls_name,[]) - h.append(node_name) - roles[cls_name] = h - return roles +import sys +from fabfile.db import cosmos_db env.user = 'root' env.timeout = 30 env.connection_attempts = 3 env.warn_only = True env.skip_bad_hosts = True -env.roledefs = _roledefs() - -#print repr(env.roledefs) +env.roledefs = cosmos_db()['members'] def all(): - env.hosts = _all_hosts() + env.hosts = cosmos_db()['members']['all'] def cosmos(): - run("cosmos update && cosmos apply"); + run("cosmos update ; cosmos -v apply"); def upgrade(): run("apt-get -qq update && apt-get -y -q dist-upgrade"); @@ -51,3 +30,6 @@ def chassis(): def newvm(fqdn,ip,domain): run("vmbuilder kvm ubuntu --domain %s --dest /var/lib/libvirt/images/%s.img --arch x86_64 --hostname %s --mem 512 --ip %s --addpkg openssh-server" % (domain,fqdn,fqdn,ip)) + +def cp(local,remote): + put(local,remote) diff --git a/fabfile/db.py b/fabfile/db.py new file mode 100644 index 0000000..129aa50 --- /dev/null +++ b/fabfile/db.py @@ -0,0 +1,45 @@ +import os +import yaml +import re + +def _all_hosts(): + return filter(lambda fn: '.' in fn and not fn.startswith('.') and os.path.isdir(fn),os.listdir(".")) + +def _load_db(): + rules = dict() + rules_file = "cosmos-rules.yaml"; + if os.path.exists(rules_file): + with open(rules_file) as fd: + rules.update(yaml.load(fd)) + + all_hosts = _all_hosts() + + members = dict() + for node_name in all_hosts: + for reg,cls in rules.iteritems(): + if re.search(reg,node_name): + for cls_name in cls.keys(): + h = members.get(cls_name,[]) + h.append(node_name) + members[cls_name] = h + members['all'] = all_hosts + + classes = dict() + for node_name in all_hosts: + node_classes = dict() + for reg,cls in rules.iteritems(): + if re.search(reg,node_name): + node_classes.update(cls) + classes[node_name] = node_classes + + return dict(classes=classes,members=members) + +_db = None +def cosmos_db(): + global _db + if _db is None: + _db = _load_db() + return _db + +if __name__ == '__main__': + print yaml.dump(cosmos_db()) diff --git a/global/overlay/etc/puppet/cosmos_enc.py b/global/overlay/etc/puppet/cosmos_enc.py index 63c3a66..131d161 100755 --- a/global/overlay/etc/puppet/cosmos_enc.py +++ b/global/overlay/etc/puppet/cosmos_enc.py @@ -5,20 +5,14 @@ import yaml import os import re -rules_path = os.environ.get("COSMOS_RULES_PATH","/etc/puppet") - node_name = sys.argv[1] -rules = dict() -for p in rules_path.split(":"): - rules_file = os.path.join(p,"cosmos-rules.yaml") - if os.path.exists(rules_file): - with open(rules_file) as fd: - rules.update(yaml.load(fd)) +db_file = os.environ.get("COSMOS_ENC_DB","/etc/puppet/cosmos-db.yaml") +db = dict(classes=dict()) -classes = dict() -for reg,cls in rules.iteritems(): - if re.search(reg,node_name): - classes.update(cls) +if os.path.exists(db_file): + with open(db_file) as fd: + db.update(yaml.load(fd)) + +print yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[])))) -print yaml.dump(dict(classes=classes)) diff --git a/global/post-tasks.d/018packages b/global/post-tasks.d/018packages index 57dff1a..bf7bf64 100755 --- a/global/post-tasks.d/018packages +++ b/global/post-tasks.d/018packages @@ -3,7 +3,6 @@ CONFIG=${CONFIG:=/etc/puppet/cosmos-modules.conf} CACHE_DIR=/var/cache/puppet-modules MODULES_DIR=${MODULES_DIR:=/etc/puppet/cosmos-modules} -GIT_TAG_PATTERN=${COSMOS_UPDATE_VERIFY_GIT_TAG_PATTERN:-multiverse*} export GNUPGHOME=/etc/cosmos/gnupg python -c "import yaml" 2>/dev/null || apt-get -y install python-yaml @@ -24,7 +23,7 @@ if [ -f $CONFIG ]; then # First pass to clone any new modules, and update those marked for updating. grep -E -v "^#" $CONFIG | ( - while read module src update; do + while read module src update pattern; do # We only support git:// urls atm if [ "${src:0:6}" = "git://" ]; then if [ ! -d $CACHE_DIR/scm/$module ]; then @@ -47,23 +46,22 @@ if [ -f $CONFIG ]; then # Second pass to verify the signatures on all modules and stage those that # have good signatures. grep -E -v "^#" $CONFIG | ( - while read module src update; do + while read module src update pattern; do # We only support git:// urls atm if [ "${src:0:6}" = "git://" ]; then # Verify git tag cd $CACHE_DIR/scm/$module - TAG=$(git tag -l $GIT_TAG_PATTERN | sort | tail -1) + TAG=$(git tag -l "${pattern:-*}" | sort | tail -1) if [ "$COSMOS_VERBOSE" = "y" ]; then echo "" echo "Checking signature on tag ${TAG} for puppet-module $module" fi if [ -z "$TAG" ]; then - echo "ERROR: No git tag found for pattern '$GIT_TAG_PATTERN' on puppet-module $module" + echo "ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module $module" continue fi - fail=1 - git tag -v $TAG > /dev/null 2>&1 && fail=0 - if [ $fail == 0 ]; then + git tag -v $TAG &> /dev/null + if [ $? == 0 ]; then if [ "$COSMOS_VERBOSE" = "y" ]; then # short output on good signature git tag -v $TAG 2>&1 | grep "gpg: Good signature" @@ -82,9 +80,9 @@ if [ -f $CONFIG ]; then # Cleanup removed puppet modules from CACHE_DIR for MODULE in $(ls -1 $CACHE_DIR/staging/); do - if ! grep -E -q "^$MODULE\s+" $CONFIG; then - rm -rf $CACHE_DIR/{scm,staging}/$MODULE - fi + if ! grep -E -q "^$MODULE\s+" $CONFIG; then + rm -rf $CACHE_DIR/{scm,staging}/$MODULE + fi done # Installing verified puppet modules