merged pull-request

Makefile

@@ -4,5 +4,9 @@ cosmos:
 upgrade:
 	fab upgrade
 
-bump:
+db:
-	./bump-tag
+	@python ./fabfile/db.py > global/overlay/etc/puppet/cosmos-db.yaml
+	@git add global/overlay/etc/puppet/cosmos-db.yaml && git commit -m "update db" global/overlay/etc/puppet/cosmos-db.yaml
+
+tag: db
+        ./bump-tag

docs/cosmos-puppet-ops.mkd

@@ -210,6 +210,14 @@ as 'ro'. The read-only remote is used by multiverse scripts during host bootstra
 # git remote add ro git://yourhost/myproj-cosmos.git
 ```
 
+Now edit .git/config and rename the 'master' branch to use the new 'origin' remote or
+you'll try to push to the multiverse remote! Finally create a branch for the 'multiverse'
+upstream so you can merge changes to multiverse:
+
+```
+# git checkout -b multiverse --track multiverse/master
+```
+
 Note that you can maintain your repo on just about any git hosting platform, including
 github, gitorius or your own local setup as long as it supports read-only "git://" access
 to your repository. It is important that the remotes called 'origin' and 'ro' refer to 

fabfile/__init__.py

@@ -1,44 +1,23 @@
 from fabric.api import run,env
-from fabric.operations import get
+from fabric.operations import get,put
 import os
 import yaml
 import re
-
+import sys
-def _all_hosts():
+from fabfile.db import cosmos_db
-   return filter(lambda fn: '.' in fn and not fn.startswith('.') and os.path.isdir(fn),os.listdir("."))
-
-def _roledefs():
-   rules = dict()
-
-   rules_file = "cosmos-rules.yaml";
-   if os.path.exists(rules_file):
-      with open(rules_file) as fd:
-         rules.update(yaml.load(fd))
-
-   roles = dict() 
-   for node_name in _all_hosts():
-      for reg,cls in rules.iteritems():
-         if re.search(reg,node_name):
-            for cls_name in cls.keys():
-               h = roles.get(cls_name,[])
-               h.append(node_name)
-               roles[cls_name] = h
-   return roles
 
 env.user = 'root'
 env.timeout = 30
 env.connection_attempts = 3
 env.warn_only = True
 env.skip_bad_hosts = True
-env.roledefs = _roledefs()
+env.roledefs = cosmos_db()['members']
-
-#print repr(env.roledefs)
 
 def all():
-    env.hosts = _all_hosts()
+    env.hosts = cosmos_db()['members']['all']
 
 def cosmos():
-    run("cosmos update && cosmos apply");
+    run("cosmos update ; cosmos -v apply");
 
 def upgrade():
     run("apt-get -qq update && apt-get -y -q dist-upgrade");
@@ -51,3 +30,6 @@ def chassis():
 
 def newvm(fqdn,ip,domain):
     run("vmbuilder kvm ubuntu --domain %s --dest /var/lib/libvirt/images/%s.img --arch x86_64 --hostname %s --mem 512 --ip %s --addpkg openssh-server" % (domain,fqdn,fqdn,ip))
+
+def cp(local,remote):
+    put(local,remote)

fabfile/db.py

@@ -0,0 +1,45 @@
+import os
+import yaml
+import re
+
+def _all_hosts():
+   return filter(lambda fn: '.' in fn and not fn.startswith('.') and os.path.isdir(fn),os.listdir("."))
+
+def _load_db():
+   rules = dict()
+   rules_file = "cosmos-rules.yaml";
+   if os.path.exists(rules_file):
+      with open(rules_file) as fd:
+         rules.update(yaml.load(fd))
+ 
+   all_hosts = _all_hosts()
+
+   members = dict()
+   for node_name in all_hosts:
+      for reg,cls in rules.iteritems():
+         if re.search(reg,node_name):
+            for cls_name in cls.keys():
+               h = members.get(cls_name,[])
+               h.append(node_name)
+               members[cls_name] = h
+   members['all'] = all_hosts
+
+   classes = dict()
+   for node_name in all_hosts:
+      node_classes = dict()
+      for reg,cls in rules.iteritems():
+         if re.search(reg,node_name):
+            node_classes.update(cls)
+      classes[node_name] = node_classes
+
+   return dict(classes=classes,members=members)
+
+_db = None
+def cosmos_db():
+   global _db
+   if _db is None:
+      _db = _load_db()
+   return _db
+
+if __name__ == '__main__':
+   print yaml.dump(cosmos_db())

global/overlay/etc/puppet/cosmos_enc.py

@@ -5,20 +5,14 @@ import yaml
 import os
 import re
 
-rules_path = os.environ.get("COSMOS_RULES_PATH","/etc/puppet")
-
 node_name = sys.argv[1]
 
-rules = dict()
+db_file = os.environ.get("COSMOS_ENC_DB","/etc/puppet/cosmos-db.yaml")
-for p in rules_path.split(":"):
+db = dict(classes=dict())
-   rules_file = os.path.join(p,"cosmos-rules.yaml")
-   if os.path.exists(rules_file):
-      with open(rules_file) as fd:
-         rules.update(yaml.load(fd))
 
-classes = dict()
+if os.path.exists(db_file):
-for reg,cls in rules.iteritems():
+   with open(db_file) as fd:
-   if re.search(reg,node_name):
+      db.update(yaml.load(fd))
-      classes.update(cls)
+
+print yaml.dump(dict(classes=db['classes'].get(node_name,dict()),parameters=dict(roles=db.get('members',[]))))
 
-print yaml.dump(dict(classes=classes))

global/post-tasks.d/018packages

@@ -3,7 +3,6 @@
 CONFIG=${CONFIG:=/etc/puppet/cosmos-modules.conf}
 CACHE_DIR=/var/cache/puppet-modules
 MODULES_DIR=${MODULES_DIR:=/etc/puppet/cosmos-modules}
-GIT_TAG_PATTERN=${COSMOS_UPDATE_VERIFY_GIT_TAG_PATTERN:-multiverse*}
 export GNUPGHOME=/etc/cosmos/gnupg
 
 python -c "import yaml" 2>/dev/null || apt-get -y install python-yaml
@@ -24,7 +23,7 @@ if [ -f $CONFIG ]; then
 
   # First pass to clone any new modules, and update those marked for updating.
   grep -E -v "^#" $CONFIG | (
-    while read module src update; do
+    while read module src update pattern; do
       # We only support git:// urls atm
       if [ "${src:0:6}" = "git://" ]; then
         if [ ! -d $CACHE_DIR/scm/$module ]; then
@@ -47,23 +46,22 @@ if [ -f $CONFIG ]; then
   # Second pass to verify the signatures on all modules and stage those that
   # have good signatures.
   grep -E -v "^#" $CONFIG | (
-    while read module src update; do
+    while read module src update pattern; do
       # We only support git:// urls atm
       if [ "${src:0:6}" = "git://" ]; then
         # Verify git tag
         cd $CACHE_DIR/scm/$module
-        TAG=$(git tag -l $GIT_TAG_PATTERN | sort | tail -1)
+        TAG=$(git tag -l "${pattern:-*}" | sort | tail -1)
         if [ "$COSMOS_VERBOSE" = "y" ]; then
           echo ""
           echo "Checking signature on tag ${TAG} for puppet-module $module"
         fi
         if [ -z "$TAG" ]; then
-          echo "ERROR: No git tag found for pattern '$GIT_TAG_PATTERN' on puppet-module $module"
+          echo "ERROR: No git tag found for pattern '${pattern:-*}' on puppet-module $module"
           continue
         fi
-        fail=1
+        git tag -v $TAG &> /dev/null
-        git tag -v $TAG > /dev/null 2>&1 && fail=0
+        if [ $? == 0 ]; then
-        if [ $fail == 0 ]; then
           if [ "$COSMOS_VERBOSE" = "y" ]; then
             # short output on good signature
             git tag -v $TAG 2>&1 | grep "gpg: Good signature"
@@ -82,9 +80,9 @@ if [ -f $CONFIG ]; then
 
   # Cleanup removed puppet modules from CACHE_DIR
   for MODULE in $(ls -1 $CACHE_DIR/staging/); do
-    if ! grep -E -q "^$MODULE\s+" $CONFIG; then
+      if ! grep -E -q "^$MODULE\s+" $CONFIG; then
-      rm -rf $CACHE_DIR/{scm,staging}/$MODULE
+          rm -rf $CACHE_DIR/{scm,staging}/$MODULE
-    fi
+      fi
   done
 
   # Installing verified puppet modules