author Micke <kano@sunet.se> 1679915017 +0000

committer Micke Nordin <kano@sunet.se> 1680110378 +0200
gpgsig -----BEGIN PGP SIGNATURE-----

 iQIzBAABCgAdFiEEIpL7dwHsMfazpY3ODaCnpXCP4lcFAmQkcyoACgkQDaCnpXCP
 4ld8Rg/7BSPA3rrDV9JbfQYtuE1/wpMy+hfYC/jIiD3VpwPKZ27r8/LzIsrquL6o
 nOfQT5cj/Jm2WrZoS+6pRlmWxzbBN6Oc6XJMxrUu+JVHSyCYNSFU+j074/4B3mNW
 +YDw2415/sEBJbhBcCmNOfWVZuOq1+AsfBkZ1vhUT0GzKahGIBMzgK8P+5IBsXCp
 7t/N5EmvFy6mzubkT8LN9ZQ0WMd2OxBeBDPUcrwlrT6kxA/1JQfaZ4PWtSb7QD0G
 IQAeUAy3p9JxXRQyWmunEYrxp0maO6EPyp6O656C75JKn7lOhMGMR2S5FT3jPK5d
 KrbewbCsRiq3O94STxytBdDjT/Ez2vk0/tj7cW5r0hb2Pd1cVgHlcRP53r8dzSn9
 RIRycGKB49k2MYByNGJN3oM9KYB2vpIhqmy9vyFn3HinhD8nyj0VDAo0LWDCKnbh
 dt7jTKAsr1y9bXPK+3YM/2QiCdPMDz1xYd1BhbFgQ5B5Eb8iB0iQn9UHPe884wNN
 Zs+yPAGRyPrAVvCnopV2IZJpGI7oKjnmcgBanbTJK+UKkfmb0a4S/fbCezjhT6Qc
 105jvtmZgj44aSqtBSkXvV9UoCtjePEBifhcQznzpi8eudbo1Hhg5UjO5rwwhl2P
 lUQgs75Rsfd549YBV/Rq1szlVODTsMhU2oDr83t8Vf/TF9GdhEg=
 =Hnvm
 -----END PGP SIGNATURE-----

Nextcloud in K8s: MVP

This is the minumum needed to run a multinode customer in kubernetes.
Each customer will run in their own namespace with a nextcloud
container and a redis container. There is a proxysql cluster running in
its own namespace, used by all customer.

This is a monorepo that uses kustomize to allow using this repo for both
prod and test (and other things we can think of in the future).

README.md

@@ -0,0 +1,2 @@
+# k8s-manifests
+

customers/base/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+  - nextcloud-configmap.yml
+  - nextcloud-deployment.yml
+  - nextcloud-ingress.yml
+  - nextcloud-service.yml
+  - redis-deployment.yml
+  - redis-service.yml
+  - s3-service.yml

customers/base/nextcloud-configmap.yml

@@ -0,0 +1,173 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nextcloud-configmap
+data:
+  000-default.conf: |
+    LoadModule remoteip_module /usr/lib/apache2/modules/mod_remoteip.so
+    LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so
+    <VirtualHost *:80>
+            ServerAdmin webmaster@localhost
+            DocumentRoot /var/www/html
+            # Log format config
+            LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" common
+            SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+
+            # Header config
+            RemoteIPHeader X-Forwarded-For
+            RemoteIPInternalProxy 37.156.195.14
+            RemoteIPInternalProxy 37.156.195.19
+            RemoteIPInternalProxy 37.156.195.84
+            RemoteIPInternalProxy 37.156.195.92
+            #ErrorDocument 404 /404.html
+            ErrorLog ${APACHE_LOG_DIR}/error.log
+            CustomLog ${APACHE_LOG_DIR}/access.log combined env=forwarded
+
+            ErrorLog ${APACHE_LOG_DIR}/error.log
+            CustomLog ${APACHE_LOG_DIR}/access.log combined
+            <Directory /var/www/html/>
+                    LimitRequestBody 0
+                    Require all granted
+                    AllowOverride All
+                    Options FollowSymLinks MultiViews
+
+                    <IfModule mod_dav.c>
+                        Dav off
+                    </IfModule>
+            </Directory>
+    </VirtualHost>
+    <VirtualHost *:443>
+            ServerAdmin webmaster@localhost
+            DocumentRoot /var/www/html
+            # Log format config
+            LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" common
+            SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+
+            # Header config
+            RemoteIPHeader X-Forwarded-For
+            RemoteIPInternalProxy 37.156.195.14
+            RemoteIPInternalProxy 37.156.195.19
+            RemoteIPInternalProxy 37.156.195.84
+            RemoteIPInternalProxy 37.156.195.92
+            #ErrorDocument 404 /404.html
+            ErrorLog ${APACHE_LOG_DIR}/error.log
+            CustomLog ${APACHE_LOG_DIR}/access.log combined env=forwarded
+            SSLEngine On
+            SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem
+            SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
+            <Directory /var/www/html/>
+                    LimitRequestBody 0
+                    Require all granted
+                    AllowOverride All
+                    Options FollowSymLinks MultiViews
+
+                    <IfModule mod_dav.c>
+                        Dav off
+                    </IfModule>
+            </Directory>
+    </VirtualHost>
+    # vim: syntax=apache ts=4 sw=4 sts=4 sr noet
+  config.php: |
+    <?php
+    $CONFIG = array (
+      'memcache.local' => '\\OC\\Memcache\\APCu',
+      'appstoreenabled' => false,
+      'apps_paths' =>
+      array (
+        0 =>
+        array (
+          'path' => '/var/www/html/apps',
+          'url' => '/apps',
+          'writable' => false,
+        ),
+        1 =>
+        array (
+          'path' => '/var/www/html/custom_apps',
+          'url' => '/custom_apps',
+          'writable' => true,
+        ),
+      ),
+      'memcache.distributed' => '\\OC\\Memcache\\Redis',
+      'memcache.locking' => '\\OC\\Memcache\\Redis',
+      'redis' =>
+      array (
+        'host' => '{{REDIS_HOST}}',
+        'port' => 6379,
+      ),
+      'forcessl' => true,
+      'overwriteprotocol' => 'https',
+      'objectstore' =>
+      array (
+        'class' => '\\OC\\Files\\ObjectStore\\S3',
+        'arguments' =>
+        array (
+          'bucket' => '{{OBJECTSTORE_S3_BUCKET}}',
+          'key' => '{{OBJECTSTORE_S3_KEY}}',
+          'secret' => '{{OBJECTSTORE_S3_SECRET}}',
+          'region' => '{{OBJECTSTORE_S3_REGION}}',
+          'hostname' => '{{OBJECTSTORE_S3_HOST}}',
+          'port' => '',
+          'objectPrefix' => 'urn:oid:',
+          'autocreate' => false,
+          'use_ssl' => true,
+          'use_path_style' => true,
+          'legacy_auth' => false,
+        ),
+      ),
+      'csrf.disabled' => true,
+      'passwordsalt' => '{{NEXTCLOUD_PASSWORDSALT}}',
+      'secret' => '{{NEXTCLOUD_SECRET}}',
+      'trusted_domains' =>
+      array (
+        0 => '{{NEXTCLOUD_TRUSTED_DOMAINS}}'
+      ),
+      'config_is_read_only' => true,
+      'datadirectory' => '/var/www/html/data',
+      'dbhost' => '{{MYSQL_HOST}}:{{MYSQL_PORT}}',
+      'dbname' => '{{MYSQL_DATABASE}}',
+      'dbpassword' => '{{MYSQL_PASSWORD}}',
+      'dbport' => '{{MYSQL_PORT}}',
+      'dbtableprefix' => 'oc_',
+      'dbtype' => 'mysql',
+      'dbuser' => '{{MYSQL_USER}}',
+      'gs.enabled' => 'true',
+      'gs.federation' => 'global',
+      'gs.trustedHosts' => ['*.sunet.se'],
+      'gss.jwt.key' => '{{GSS_JWT_KEY}}',
+      'gss.master.url' => '{{GSS_MASTER_URL}}',
+      'gss.mode' => 'slave',
+      'gss.user.discovery.module' => '\\OCA\\GlobalSiteSelector\\UserDiscoveryModules\\ManualUserMapping',
+      'installed' => true,
+      'instanceid' => '{{NEXTCLOUD_INSTANCEID}}',
+      'log_type' => 'file',
+      'loglevel' => 0,
+      'lookup_server' => '{{LOOKUP_SERVER}}',
+      'mail_domain' => '{{MAIL_DOMAIN}}',
+      'mail_from_address' => '{{MAIL_FROM_ADDRESS}}',
+      'mail_sendmailmode' => 'smtp',
+      'mail_smtpauth' => 1,
+      'mail_smtpauthtype' => 'LOGIN',
+      'mail_smtphost' => '{{MAIL_SMTPHOST}}',
+      'mail_smtpmode' => 'smtp',
+      'mail_smtpname' => '{{MAIL_SMTPNAME}}',
+      'mail_smtppassword' => '{{MAIL_SMTPPASSWORD}}',
+      'mail_smtpport' => '587',
+      'mail_smtpsecure' => 'tls',
+      'mail_template_class' => 'OCA\DriveEmailTemplate\EMailTemplate',
+      'mysql.utf8mb4' => true,
+      'overwritehost' => '{{SITE_NAME}}',
+      'overwrite.cli.url' => 'https://{{SITE_NAME}}',
+      'templatedirectory' => '',
+      'skeletondirectory' => '',
+      'twofactor_enforced' => 'true',
+      'twofactor_enforced_groups' =>
+      array (
+        0 => 'admin',
+      ),
+      'version' => '{{NEXTCLOUD_VERSION_STRING}}',
+      'app_install_overwrite' =>
+      array (
+        0 => 'globalsiteselector',
+      ),
+
+    );

customers/base/nextcloud-deployment.yml

@@ -0,0 +1,180 @@
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: customer-node
+  labels:
+    app: customer-node
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: customer-node
+  updateStrategy:
+    type: RollingUpdate
+  volumeClaimTemplates:
+    - metadata:
+        name: nextcloud-data
+      spec:
+        storageClassName: mayastor-3
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+  template:
+    metadata:
+      labels:
+        app: customer-node
+    spec:
+      restartPolicy: Always
+      containers:
+        - name: customer
+          image: docker.sunet.se/drive/nextcloud-custom:25.0.3.3-4
+          volumeMounts:
+            - name: nextcloud-data
+              mountPath: /var/www/html/config
+              subPath: config
+            - name: default-config
+              mountPath: /etc/apache2/sites-enabled/000-default.conf
+              subPath: 000-default.conf
+            - name: hugepage
+              mountPath: /dev/hugepages
+          resources:
+            limits:
+              cpu: 1500m
+              hugepages-2Mi: 2048Mi
+              memory: 2048Mi
+            requests:
+              cpu: 500m
+              hugepages-2Mi: 2048Mi
+              memory: 256Mi
+          readinessProbe:
+            tcpSocket:
+              port: 80
+            initialDelaySeconds: 10
+            periodSeconds: 60
+          livenessProbe:
+            tcpSocket:
+              port: 80
+            initialDelaySeconds: 20
+            periodSeconds: 180
+          ports:
+            - containerPort: 80
+              name: nextcloud-http
+          command: ["/bin/bash"]
+          args: ["-c", "apachectl -D FOREGROUND"]
+      initContainers:
+        - image: docker.sunet.se/sunet/docker-jinja:latest
+          name: init-config
+          volumeMounts:
+            - name: nextcloud-config-template
+              mountPath: /tmp/config.php.template
+              subPath: config.php
+            - name: nextcloud-data
+              mountPath: /var/www/html/config
+              subPath: config
+          env:
+            - name: GSS_MASTER_URL
+              value: "https://drive.test.sunet.se"
+            - name: GSS_JWT_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: gss-secret
+                  key: "jwt_key"
+            - name: LOOKUP_SERVER
+              value: "https://lookup.drive.test.sunet.se"
+            - name: MAIL_DOMAIN
+              value: "drive.test.sunet.se"
+            - name: MAIL_FROM_ADDRESS
+              value: "noreply"
+            - name: MAIL_SMTPHOST
+              value: "smtp.sunet.se"
+            - name: MAIL_SMTPNAME
+              value: "noreply@drive.test.sunet.se"
+            - name: MAIL_SMTPPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mail-secret
+                  key: "smtp_password"
+            - name: MYSQL_DATABASE
+              value: "nextcloud_customer"
+            - name: MYSQL_USER
+              value: "nextcloud_customer"
+            - name: MYSQL_HOST
+              value: "proxysqlcluster.proxysql"
+            - name: MYSQL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: db-secret
+                  key: "db_password"
+            - name: MYSQL_PORT
+              value: "6033"
+            - name: NEXTCLOUD_TRUSTED_DOMAINS
+              value: "customer.drive.test.sunet.se"
+            - name: NEXTCLOUD_ADMIN_USER
+              value: admin
+            - name: NEXTCLOUD_VERSION_STRING
+              value: "25.0.3.3"
+            - name: NEXTCLOUD_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: nc-secret
+                  key: "nc_admin_password"
+            - name: NEXTCLOUD_PASSWORDSALT
+              valueFrom:
+                secretKeyRef:
+                  name: nc-secret
+                  key: "nc_passwordsalt"
+            - name: NEXTCLOUD_INSTANCEID
+              valueFrom:
+                secretKeyRef:
+                  name: nc-secret
+                  key: "nc_instanceid"
+            - name: NEXTCLOUD_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: nc-secret
+                  key: "nc_secret"
+            - name: OBJECTSTORE_S3_REGION
+              value: "us-east-1"
+            - name: OBJECTSTORE_S3_HOST
+              value: "s3.sto4.safedc.net"
+            - name: OBJECTSTORE_S3_BUCKET
+              value: "primary-customer-drive-test.sunet.se"
+            - name: OBJECTSTORE_S3_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: s3-secret
+                  key: "s3_key"
+            - name: OBJECTSTORE_S3_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: s3-secret
+                  key: "s3_secret"
+            - name: OBJECTSTORE_S3_USEPATH_STYLE
+              value: "true"
+            - name: OBJECTSTORE_S3_AUTOCREATE
+              value: "true"
+            - name: OBJECTSTORE_S3_SSL
+              value: "true"
+            - name: REDIS_HOST
+              value: "redis"
+            - name: SITE_NAME
+              value: "customer.drive.test.sunet.se"
+          command: ["/bin/bash", "-c", "/usr/bin/j2 -f env -o /var/www/html/config/config.php /tmp/config.php.template"]
+      volumes:
+        - name: nextcloud-config-template
+          configMap:
+            name: nextcloud-configmap
+            items:
+              - key: "config.php"
+                path: "config.php"
+        - name: default-config
+          configMap:
+            name: nextcloud-configmap
+            items:
+              - key: "000-default.conf"
+                path: "000-default.conf"
+        - name: hugepage
+          emptyDir:
+            medium: HugePages

customers/base/nextcloud-ingress.yml

@@ -0,0 +1,31 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: customer-ingress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  defaultBackend:
+    service:
+      name: customer-node
+      port:
+        number: 8443
+  tls:
+    - hosts:
+        - customer.drive.test.sunet.se 
+      secretName: tls-secret
+  
+  rules:
+  - host: customer.drive.test.sunet.se
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend: 
+          service:
+            name: customer-node
+            port:
+              number: 80

customers/base/nextcloud-service.yml

@@ -0,0 +1,24 @@
+---
+apiVersion: v1
+items:
+- apiVersion: v1
+  kind: Service
+  metadata:
+    labels:
+      app: customer-node
+    name: customer-node
+  spec:
+    ports:
+    - port: 80
+      protocol: TCP
+      targetPort: 80
+    selector:
+      app: customer-node
+    sessionAffinity: None
+    type: ClusterIP
+  status:
+    loadBalancer: {}
+kind: List
+metadata:
+  resourceVersion: ""
+  selfLink: ""

customers/base/redis-deployment.yml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+  labels:
+    app: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: default
+        image: redis:latest
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 6379
+        args:
+        - "--save"
+        - ""
+        - "--appendonly"
+        - "no"
+

customers/base/redis-service.yml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+  labels:
+    app: redis
+  name: redis
+spec:
+  ports:
+  - name: redis-port
+    nodePort: 30379
+    port: 6379
+    protocol: TCP
+    targetPort: 6379
+  selector:
+    app: redis
+  type: NodePort

customers/base/s3-service.yml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sto4
+spec:
+  type: ExternalName
+  externalName: s3.sto4.safedc.net
+  ports:
+    - port: 443

customers/overlays/vr/test/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: 'kustomize.config.k8s.io/v1beta1'
+kind: Kustomization
+bases:
+- ../../../base
+patchesStrategicMerge:
+  - nextcloud-deployment.yml
+  - nextcloud-ingress.yml

customers/overlays/vr/test/nextcloud-deployment.yml

@@ -0,0 +1,34 @@
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: customer-node
+  labels:
+    app: customer-node
+spec:
+  template:
+    metadata:
+      labels:
+        app: customer-node
+    spec:
+      initContainers:
+        - image: docker.sunet.se/sunet/docker-jinja:latest
+          name: init-config
+          env:
+            - name: MYSQL_DATABASE
+              value: "nextcloud_vr"
+            - name: MYSQL_USER
+              value: "nextcloud_vr"
+            - name: GSS_MASTER_URL
+              value: "https://drive.test.sunet.se"
+            - name: LOOKUP_SERVER
+              value: "https://lookup.drive.test.sunet.se"
+            - name: MAIL_DOMAIN
+              value: "drive.test.sunet.se"
+            - name: MAIL_SMTPNAME
+              value: "noreply@drive.test.sunet.se"
+            - name: NEXTCLOUD_TRUSTED_DOMAINS
+              value: "vr.drive.test.sunet.se"
+            - name: OBJECTSTORE_S3_BUCKET
+              value: "primary-vr-drive-test.sunet.se"
+            - name: SITE_NAME
+              value: "vr.drive.test.sunet.se"

customers/overlays/vr/test/nextcloud-ingress.yml

@@ -0,0 +1,26 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: customer-ingress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  tls:
+    - hosts:
+        - vr.drive.test.sunet.se 
+      secretName: tls-secret
+  
+  rules:
+  - host: vr.drive.test.sunet.se
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend: 
+          service:
+            name: customer-node
+            port:
+              number: 80

proxysql/base/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - proxysql-configmap.yml
+  - proxysql-db-service.yml
+  - proxysql-deployment.yml
+  - proxysql-namespace.yml
+  - proxysql-service.yml

proxysql/base/proxysql-configmap.yml

@@ -0,0 +1,408 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: proxysql-configmap
+  namespace: proxysql
+data: 
+  proxysql.cnf: |
+    datadir="/var/lib/proxysql"
+
+    admin_variables=
+    {
+        admin_credentials="admin:{{ADMIN_PASSWORD}};cluster:{{CLUSTER_PASSWORD}}"
+        mysql_ifaces="0.0.0.0:6032"
+        refresh_interval=2000
+        cluster_username="cluster"
+        cluster_password="{{CLUSTER_PASSWORD}}"
+    }
+
+    mysql_variables=
+    {
+        threads=4
+        max_connections=2048
+        default_query_delay=0
+        default_query_timeout=36000000
+        have_compress=true
+        poll_timeout=2000
+        interfaces="0.0.0.0:6033;/tmp/proxysql.sock"
+        default_schema="information_schema"
+        stacksize=1048576
+        server_version="8.0.23"
+        connect_timeout_server=3000
+        monitor_username="proxysql"
+        monitor_password="{{MONITOR_PASSWORD}}"
+        monitor_history=600000
+        monitor_connect_interval=60000
+        monitor_ping_interval=10000
+        monitor_read_only_interval=1500
+        monitor_read_only_timeout=500
+        ping_interval_server_msec=120000
+        ping_timeout_server=500
+        commands_stats=true
+        sessions_sort=true
+        connect_retries_on_failure=10
+    }
+
+    mysql_galera_hostgroups =
+    (
+        {
+
+            writer_hostgroup=10
+            backup_writer_hostgroup=20
+            reader_hostgroup=30
+            offline_hostgroup=9999
+            max_writers=1
+            writer_is_also_reader=2
+            max_transactions_behind=30
+            active=1
+        }
+    )
+    mysql_query_rules =
+    (
+        {
+            rule_id=100
+            active=1
+            match_pattern="^SELECT .* FOR UPDATE"
+            destination_hostgroup=10
+            apply=1
+        },
+        {
+            rule_id=200
+            active=1
+            match_pattern="^SELECT .*"
+            destination_hostgroup=30
+            apply=1
+        },
+        {
+            rule_id=300
+            active=1
+            match_pattern=".*"
+            destination_hostgroup=10
+            apply=1
+        }
+    )
+    mysql_servers =
+    (
+        { address="multinode-db1" , port=3306 , hostgroup=10, max_connections=100 },
+        { address="multinode-db2" , port=3306 , hostgroup=20, max_connections=100 },
+        { address="multinode-db3" , port=3306 , hostgroup=30, max_connections=100 }
+    )
+
+    mysql_users =
+    (
+      {
+          username="nextcloud_antagning"
+          password="{{ANTAGNING_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_bth"
+          password="{{BTH_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_chalmers"
+          password="{{CHALMERS_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_du"
+          password="{{DU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_esh"
+          password="{{ESH_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_fhs"
+          password="{{FHS_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_gu"
+          password="{{GU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_hb"
+          password="{{HB_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_hhs"
+          password="{{HHS_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_hig"
+          password="{{HIG_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_his"
+          password="{{HIS_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_hj"
+          password="{{HJ_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_hv"
+          password="{{HV_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_irf"
+          password="{{IRF_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_kb"
+          password="{{KB_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_ki"
+          password="{{KI_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_kkh"
+          password="{{KKH_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_kmh"
+          password="{{KMH_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_konstfack"
+          password="{{KONSTFACK_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_kth"
+          password="{{KTH_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_kva"
+          password="{{KVA_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_liu"
+          password="{{LIU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_lnu"
+          password="{{LNU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_ltu"
+          password="{{LTU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_lu"
+          password="{{LU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_mau"
+          password="{{MAU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_miun"
+          password="{{MIUN_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_nordunet"
+          password="{{NORDUNET_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_nrm"
+          password="{{NRM_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_oru"
+          password="{{ORU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_rkh"
+          password="{{RKH_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_shh"
+          password="{{SHH_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_sics"
+          password="{{SICS_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_slu"
+          password="{{SLU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_smhi"
+          password="{{SMHI_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_sp"
+          password="{{SP_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_swamid"
+          password="{{SWAMID_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_ths"
+          password="{{THS_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_uhr"
+          password="{{UHR_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_umu"
+          password="{{UMU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_uniarts"
+          password="{{UNIARTS_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_uu"
+          password="{{UU_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_vinnova"
+          password="{{VINNOVA_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+      {
+          username="nextcloud_vr"
+          password="{{VR_PASSWORD}}"
+          default_hostgroup=10
+          transaction_persistent=1
+          active=1
+      },
+    )
+
+    proxysql_servers =
+    (
+        { hostname = "proxysql-0.proxysqlcluster", port = 6032, weight = 1 },
+        { hostname = "proxysql-1.proxysqlcluster", port = 6032, weight = 1 },
+        { hostname = "proxysql-2.proxysqlcluster", port = 6032, weight = 1 }
+    )

proxysql/base/proxysql-db-service.yml

@@ -0,0 +1,33 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db1
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db1.drive.test.sunet.se
+  ports:
+    - port: 3306
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db2
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db2.drive.test.sunet.se
+  ports:
+    - port: 3306
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db3
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db3.drive.test.sunet.se
+  ports:
+    - port: 3306

proxysql/base/proxysql-deployment.yml

@@ -0,0 +1,311 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: proxysql
+  namespace: proxysql
+  labels:
+    app: proxysql
+spec:
+  replicas: 3
+  serviceName: proxysqlcluster
+  selector:
+    matchLabels:
+      app: proxysql
+  updateStrategy:
+    type: RollingUpdate
+  volumeClaimTemplates:
+    - metadata:
+        name: proxysql-data
+        namespace: proxysql
+      spec:
+        storageClassName: mayastor-3
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 2Gi
+    - metadata:
+        name: proxysql-etc
+        namespace: proxysql
+      spec:
+        storageClassName: mayastor-3
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 2Gi
+  template:
+    metadata:
+      labels:
+        app: proxysql
+    spec:
+      restartPolicy: Always
+      containers:
+        - image: docker.sunet.se/drive/proxysql:latest
+          name: proxysql
+          volumeMounts:
+            - name: proxysql-data
+              mountPath: /var/lib/proxysql
+              subPath: data
+            - name: proxysql-etc
+              mountPath: /etc
+              subPath: etc
+          ports:
+            - containerPort: 6033
+              name: proxysql-mysql
+            - containerPort: 6032
+              name: proxysql-admin
+      initContainers:
+        - image: docker.sunet.se/sunet/docker-jinja:latest
+          name: init-config
+          volumeMounts:
+            - name: proxysql-config-template
+              mountPath: /tmp/proxysql.cnf.template
+              subPath: proxysql.cnf
+            - name: proxysql-etc
+              mountPath: /etc
+              subPath: etc
+          env:
+            - name: ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_admin_password"
+            - name: CLUSTER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_cluster_password"
+            - name: MONITOR_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_monitor_password"
+            - name: ANTAGNING_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_antagning_password"
+            - name: BTH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_bth_password"
+            - name: CHALMERS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_chalmers_password"
+            - name: DU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_du_password"
+            - name: ESH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_esh_password"
+            - name: FHS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_fhs_password"
+            - name: GU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_gu_password"
+            - name: HB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_hb_password"
+            - name: HHS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_hhs_password"
+            - name: HIG_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_hig_password"
+            - name: HIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_his_password"
+            - name: HJ_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_hj_password"
+            - name: HV_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_hv_password"
+            - name: IRF_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_irf_password"
+            - name: KB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_kb_password"
+            - name: KI_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_ki_password"
+            - name: KKH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_kkh_password"
+            - name: KMH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_kmh_password"
+            - name: KONSTFACK_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_konstfack_password"
+            - name: KTH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_kth_password"
+            - name: KVA_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_kva_password"
+            - name: LIU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_liu_password"
+            - name: LNU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_lnu_password"
+            - name: LTU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_ltu_password"
+            - name: LU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_lu_password"
+            - name: MAU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_mau_password"
+            - name: MIUN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_miun_password"
+            - name: NORDUNET_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_nordunet_password"
+            - name: NRM_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_nrm_password"
+            - name: ORU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_oru_password"
+            - name: RKH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_rkh_password"
+            - name: SHH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_shh_password"
+            - name: SICS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_sics_password"
+            - name: SLU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_slu_password"
+            - name: SMHI_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_smhi_password"
+            - name: SP_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_sp_password"
+            - name: SWAMID_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_swamid_password"
+            - name: THS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_ths_password"
+            - name: UHR_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_uhr_password"
+            - name: UMU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_umu_password"
+            - name: UNIARTS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_uniarts_password"
+            - name: UU_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_uu_password"
+            - name: VINNOVA_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_vinnova_password"
+            - name: VR_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: proxysql-secret
+                  key: "proxysql_vr_password"
+          command: ["/bin/sh", "-c", "/usr/bin/j2 -f env -o /etc/proxysql.cnf /tmp/proxysql.cnf.template"]
+      volumes:
+        - name: proxysql-config-template
+          configMap:
+            name: proxysql-configmap
+            items:
+              - key: "proxysql.cnf"
+                path: "proxysql.cnf"

proxysql/base/proxysql-namespace.yml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: proxysql
+spec:
+  finalizers:
+  - kubernetes

proxysql/base/proxysql-service.yml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+  labels:
+    app: proxysql
+  name: proxysql
+  namespace: proxysql
+spec:
+  ports:
+  - name: proxysql-mysql
+    nodePort: 30033
+    port: 6033
+    protocol: TCP
+    targetPort: 6033
+  - name: proxysql-admin
+    nodePort: 30032
+    port: 6032
+    protocol: TCP
+    targetPort: 6032
+  selector:
+    app: proxysql
+  type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: proxysqlcluster
+  namespace: proxysql
+  labels:
+    app: proxysql
+spec:
+  clusterIP: None
+  ports:
+  - port: 6032
+    name: proxysql-admin
+  selector:
+    app: proxysql

proxysql/overlays/prod/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: 'kustomize.config.k8s.io/v1beta1'
+kind: Kustomization
+bases:
+- ../../base
+patchesStrategicMerge:
+  - proxysql-db-service.yml

proxysql/overlays/prod/proxysql-db-service.yml

@@ -0,0 +1,33 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db1
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db1.drive.test.sunet.se
+  ports:
+    - port: 3306
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db2
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db2.drive.test.sunet.se
+  ports:
+    - port: 3306
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db3
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db3.drive.test.sunet.se
+  ports:
+    - port: 3306

proxysql/overlays/test/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: 'kustomize.config.k8s.io/v1beta1'
+kind: Kustomization
+bases:
+- ../../base
+patchesStrategicMerge:
+  - proxysql-db-service.yml

proxysql/overlays/test/proxysql-db-service.yml

@@ -0,0 +1,33 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db1
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db1.drive.test.sunet.se
+  ports:
+    - port: 3306
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db2
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db2.drive.test.sunet.se
+  ports:
+    - port: 3306
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: multinode-db3
+  namespace: proxysql
+spec:
+  type: ExternalName
+  externalName: multinode-db3.drive.test.sunet.se
+  ports:
+    - port: 3306