diff --git a/global/overlay/etc/puppet/modules/cdn/files/set-provisioner-pw b/global/overlay/etc/puppet/modules/cdn/files/set-provisioner-pw
new file mode 100755
index 0000000..3392a6b
--- /dev/null
+++ b/global/overlay/etc/puppet/modules/cdn/files/set-provisioner-pw
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# When initializing step-ca with the docker flag STEPCA_INIT_PASSWORD_FILE the
+# password will be used both for key encryption as well as the admin "step"
+# provisioner. If not using that flag a separate password will be generated for
+# each usage. This seems better as you are not typing the encryption password
+# any other time, while the provisioner password is used anytime you are
+# managing things.
+#
+# This script is used on first setup of step-ca to modify the provisioner to
+# use its own password instead. Pending
+# https://github.com/smallstep/cli/pull/1294 you need to supply the new
+# password by hand, see below for the commented out command.
+
+# If we detect that the key-password is still valid for the provisioner change it.
+if step ca admin list --admin-subject=step --admin-password-file=/opt/step-ca/init/secrets/key-password > /dev/null 2>&1; then
+  echo "admin provisioner still using encryption key password, changing it"
+
+  # Change the provisioner password
+  # https://smallstep.com/docs/step-ca/provisioners/#changing-a-jwk-provisioner-password
+  OLD_ENCRYPTED_KEY=$(step ca provisioner list | jq -r '.[] | select(.name == "admin").encryptedKey')
+
+  # Unfortunately not possible to supply --password-file to "crypto jwe encrypt" yet, pending https://github.com/smallstep/cli/pull/1294
+  #ENCRYPTED_KEY=$(echo $OLD_ENCRYPTED_KEY | step crypto jwe decrypt --password-file /opt/step-ca/init/secrets/key-password | step crypto jwe encrypt --alg PBES2-HS256+A128KW --password-file /opt/step-ca/init/secrets/provisioner-password | step crypto jose format)
+  ENCRYPTED_KEY=$(echo "$OLD_ENCRYPTED_KEY" | step crypto jwe decrypt --password-file /opt/step-ca/init/secrets/key-password | step crypto jwe encrypt --alg PBES2-HS256+A128KW | step crypto jose format)
+
+  step ca provisioner update admin --private-key=<(echo -n "$ENCRYPTED_KEY") --admin-subject=step --admin-password-file=/opt/step-ca/init/secrets/key-password
+
+  # Now verify that we can use the expected password for the admin provisioner
+  if ! step ca admin list --admin-subject=step --admin-password-file=/opt/step-ca/init/secrets/provisioner-password > /dev/null 2>&1; then
+    echo "tried updating provisioner password but it does not work!"
+    exit 1
+  fi
+else
+  echo "admin provisioner not using encryption key password, doing nothing"
+fi
diff --git a/global/overlay/etc/puppet/modules/cdn/manifests/ca.pp b/global/overlay/etc/puppet/modules/cdn/manifests/ca.pp
index bae46da..f3a7815 100644
--- a/global/overlay/etc/puppet/modules/cdn/manifests/ca.pp
+++ b/global/overlay/etc/puppet/modules/cdn/manifests/ca.pp
@@ -36,6 +36,21 @@ class cdn::ca(
     mode   => '0750',
   }
 
+  file { '/opt/step-ca/init/scripts':
+    ensure => directory,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+
+  file { '/opt/step-ca/init/scripts/set-provisioner-pw':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    content => file('cdn/ca/set-provisioner-pw'),
+  }
+
   if $ca_secrets {
     if $ca_secrets['key_password'] {
       file { '/opt/step-ca/init/secrets/key-password':