From 1c192c41fd1d32cc33cf4a1e0c012c45a0b2acf5 Mon Sep 17 00:00:00 2001 From: Micke Date: Mon, 27 Mar 2023 11:03:37 +0000 Subject: [PATCH] Nextcloud in K8s: MVP This is the minumum needed to run a multinode customer in kubernetes. Each customer will run in their own namespace with a nextcloud container and a redis container. There is a proxysql cluster running in its own namespace, used by all customer. This is a monorepo that uses kustomize to allow using this repo for both prod and test (and other things we can think of in the future). --- README.md | 2 + customers/base/kustomization.yaml | 8 + customers/base/nextcloud-configmap.yml | 173 ++++++++ customers/base/nextcloud-deployment.yml | 180 ++++++++ customers/base/nextcloud-ingress.yml | 31 ++ customers/base/nextcloud-service.yml | 24 ++ customers/base/redis-deployment.yml | 28 ++ customers/base/redis-service.yml | 17 + customers/base/s3-service.yml | 10 + customers/overlays/vr/test/kustomization.yaml | 7 + .../overlays/vr/test/nextcloud-deployment.yml | 34 ++ .../overlays/vr/test/nextcloud-ingress.yml | 26 ++ proxysql/base/kustomization.yaml | 6 + proxysql/base/proxysql-configmap.yml | 408 ++++++++++++++++++ proxysql/base/proxysql-db-service.yml | 33 ++ proxysql/base/proxysql-deployment.yml | 311 +++++++++++++ proxysql/base/proxysql-namespace.yml | 7 + proxysql/base/proxysql-service.yml | 38 ++ proxysql/overlays/prod/kustomization.yaml | 6 + .../overlays/prod/proxysql-db-service.yml | 33 ++ proxysql/overlays/test/kustomization.yaml | 6 + .../overlays/test/proxysql-db-service.yml | 33 ++ 22 files changed, 1421 insertions(+) create mode 100644 README.md create mode 100644 customers/base/kustomization.yaml create mode 100644 customers/base/nextcloud-configmap.yml create mode 100644 customers/base/nextcloud-deployment.yml create mode 100644 customers/base/nextcloud-ingress.yml create mode 100644 customers/base/nextcloud-service.yml create mode 100644 customers/base/redis-deployment.yml create mode 100644 customers/base/redis-service.yml create mode 100644 customers/base/s3-service.yml create mode 100644 customers/overlays/vr/test/kustomization.yaml create mode 100644 customers/overlays/vr/test/nextcloud-deployment.yml create mode 100644 customers/overlays/vr/test/nextcloud-ingress.yml create mode 100644 proxysql/base/kustomization.yaml create mode 100644 proxysql/base/proxysql-configmap.yml create mode 100644 proxysql/base/proxysql-db-service.yml create mode 100644 proxysql/base/proxysql-deployment.yml create mode 100644 proxysql/base/proxysql-namespace.yml create mode 100644 proxysql/base/proxysql-service.yml create mode 100644 proxysql/overlays/prod/kustomization.yaml create mode 100644 proxysql/overlays/prod/proxysql-db-service.yml create mode 100644 proxysql/overlays/test/kustomization.yaml create mode 100644 proxysql/overlays/test/proxysql-db-service.yml diff --git a/README.md b/README.md new file mode 100644 index 0000000..dc019bc --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# k8s-manifests + diff --git a/customers/base/kustomization.yaml b/customers/base/kustomization.yaml new file mode 100644 index 0000000..a86d006 --- /dev/null +++ b/customers/base/kustomization.yaml @@ -0,0 +1,8 @@ +resources: + - nextcloud-configmap.yml + - nextcloud-deployment.yml + - nextcloud-ingress.yml + - nextcloud-service.yml + - redis-deployment.yml + - redis-service.yml + - s3-service.yml diff --git a/customers/base/nextcloud-configmap.yml b/customers/base/nextcloud-configmap.yml new file mode 100644 index 0000000..cbaf947 --- /dev/null +++ b/customers/base/nextcloud-configmap.yml @@ -0,0 +1,173 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nextcloud-configmap +data: + 000-default.conf: | + LoadModule remoteip_module /usr/lib/apache2/modules/mod_remoteip.so + LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + # Log format config + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" common + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + + # Header config + RemoteIPHeader X-Forwarded-For + RemoteIPInternalProxy 37.156.195.14 + RemoteIPInternalProxy 37.156.195.19 + RemoteIPInternalProxy 37.156.195.84 + RemoteIPInternalProxy 37.156.195.92 + #ErrorDocument 404 /404.html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined env=forwarded + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + LimitRequestBody 0 + Require all granted + AllowOverride All + Options FollowSymLinks MultiViews + + + Dav off + + + + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + # Log format config + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" common + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + + # Header config + RemoteIPHeader X-Forwarded-For + RemoteIPInternalProxy 37.156.195.14 + RemoteIPInternalProxy 37.156.195.19 + RemoteIPInternalProxy 37.156.195.84 + RemoteIPInternalProxy 37.156.195.92 + #ErrorDocument 404 /404.html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined env=forwarded + SSLEngine On + SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem + SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key + + LimitRequestBody 0 + Require all granted + AllowOverride All + Options FollowSymLinks MultiViews + + + Dav off + + + + # vim: syntax=apache ts=4 sw=4 sts=4 sr noet + config.php: | + '\\OC\\Memcache\\APCu', + 'appstoreenabled' => false, + 'apps_paths' => + array ( + 0 => + array ( + 'path' => '/var/www/html/apps', + 'url' => '/apps', + 'writable' => false, + ), + 1 => + array ( + 'path' => '/var/www/html/custom_apps', + 'url' => '/custom_apps', + 'writable' => true, + ), + ), + 'memcache.distributed' => '\\OC\\Memcache\\Redis', + 'memcache.locking' => '\\OC\\Memcache\\Redis', + 'redis' => + array ( + 'host' => '{{REDIS_HOST}}', + 'port' => 6379, + ), + 'forcessl' => true, + 'overwriteprotocol' => 'https', + 'objectstore' => + array ( + 'class' => '\\OC\\Files\\ObjectStore\\S3', + 'arguments' => + array ( + 'bucket' => '{{OBJECTSTORE_S3_BUCKET}}', + 'key' => '{{OBJECTSTORE_S3_KEY}}', + 'secret' => '{{OBJECTSTORE_S3_SECRET}}', + 'region' => '{{OBJECTSTORE_S3_REGION}}', + 'hostname' => '{{OBJECTSTORE_S3_HOST}}', + 'port' => '', + 'objectPrefix' => 'urn:oid:', + 'autocreate' => false, + 'use_ssl' => true, + 'use_path_style' => true, + 'legacy_auth' => false, + ), + ), + 'csrf.disabled' => true, + 'passwordsalt' => '{{NEXTCLOUD_PASSWORDSALT}}', + 'secret' => '{{NEXTCLOUD_SECRET}}', + 'trusted_domains' => + array ( + 0 => '{{NEXTCLOUD_TRUSTED_DOMAINS}}' + ), + 'config_is_read_only' => true, + 'datadirectory' => '/var/www/html/data', + 'dbhost' => '{{MYSQL_HOST}}:{{MYSQL_PORT}}', + 'dbname' => '{{MYSQL_DATABASE}}', + 'dbpassword' => '{{MYSQL_PASSWORD}}', + 'dbport' => '{{MYSQL_PORT}}', + 'dbtableprefix' => 'oc_', + 'dbtype' => 'mysql', + 'dbuser' => '{{MYSQL_USER}}', + 'gs.enabled' => 'true', + 'gs.federation' => 'global', + 'gs.trustedHosts' => ['*.sunet.se'], + 'gss.jwt.key' => '{{GSS_JWT_KEY}}', + 'gss.master.url' => '{{GSS_MASTER_URL}}', + 'gss.mode' => 'slave', + 'gss.user.discovery.module' => '\\OCA\\GlobalSiteSelector\\UserDiscoveryModules\\ManualUserMapping', + 'installed' => true, + 'instanceid' => '{{NEXTCLOUD_INSTANCEID}}', + 'log_type' => 'file', + 'loglevel' => 0, + 'lookup_server' => '{{LOOKUP_SERVER}}', + 'mail_domain' => '{{MAIL_DOMAIN}}', + 'mail_from_address' => '{{MAIL_FROM_ADDRESS}}', + 'mail_sendmailmode' => 'smtp', + 'mail_smtpauth' => 1, + 'mail_smtpauthtype' => 'LOGIN', + 'mail_smtphost' => '{{MAIL_SMTPHOST}}', + 'mail_smtpmode' => 'smtp', + 'mail_smtpname' => '{{MAIL_SMTPNAME}}', + 'mail_smtppassword' => '{{MAIL_SMTPPASSWORD}}', + 'mail_smtpport' => '587', + 'mail_smtpsecure' => 'tls', + 'mail_template_class' => 'OCA\DriveEmailTemplate\EMailTemplate', + 'mysql.utf8mb4' => true, + 'overwritehost' => '{{SITE_NAME}}', + 'overwrite.cli.url' => 'https://{{SITE_NAME}}', + 'templatedirectory' => '', + 'skeletondirectory' => '', + 'twofactor_enforced' => 'true', + 'twofactor_enforced_groups' => + array ( + 0 => 'admin', + ), + 'version' => '{{NEXTCLOUD_VERSION_STRING}}', + 'app_install_overwrite' => + array ( + 0 => 'globalsiteselector', + ), + + ); diff --git a/customers/base/nextcloud-deployment.yml b/customers/base/nextcloud-deployment.yml new file mode 100644 index 0000000..35cf8b3 --- /dev/null +++ b/customers/base/nextcloud-deployment.yml @@ -0,0 +1,180 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: customer-node + labels: + app: customer-node +spec: + replicas: 1 + selector: + matchLabels: + app: customer-node + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - metadata: + name: nextcloud-data + spec: + storageClassName: mayastor-3 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + template: + metadata: + labels: + app: customer-node + spec: + restartPolicy: Always + containers: + - name: customer + image: docker.sunet.se/drive/nextcloud-custom:25.0.3.3-4 + volumeMounts: + - name: nextcloud-data + mountPath: /var/www/html/config + subPath: config + - name: default-config + mountPath: /etc/apache2/sites-enabled/000-default.conf + subPath: 000-default.conf + - name: hugepage + mountPath: /dev/hugepages + resources: + limits: + cpu: 1500m + hugepages-2Mi: 2048Mi + memory: 2048Mi + requests: + cpu: 500m + hugepages-2Mi: 2048Mi + memory: 256Mi + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 10 + periodSeconds: 60 + livenessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 20 + periodSeconds: 180 + ports: + - containerPort: 80 + name: nextcloud-http + command: ["/bin/bash"] + args: ["-c", "apachectl -D FOREGROUND"] + initContainers: + - image: docker.sunet.se/sunet/docker-jinja:latest + name: init-config + volumeMounts: + - name: nextcloud-config-template + mountPath: /tmp/config.php.template + subPath: config.php + - name: nextcloud-data + mountPath: /var/www/html/config + subPath: config + env: + - name: GSS_MASTER_URL + value: "https://drive.test.sunet.se" + - name: GSS_JWT_KEY + valueFrom: + secretKeyRef: + name: gss-secret + key: "jwt_key" + - name: LOOKUP_SERVER + value: "https://lookup.drive.test.sunet.se" + - name: MAIL_DOMAIN + value: "drive.test.sunet.se" + - name: MAIL_FROM_ADDRESS + value: "noreply" + - name: MAIL_SMTPHOST + value: "smtp.sunet.se" + - name: MAIL_SMTPNAME + value: "noreply@drive.test.sunet.se" + - name: MAIL_SMTPPASSWORD + valueFrom: + secretKeyRef: + name: mail-secret + key: "smtp_password" + - name: MYSQL_DATABASE + value: "nextcloud_customer" + - name: MYSQL_USER + value: "nextcloud_customer" + - name: MYSQL_HOST + value: "proxysqlcluster.proxysql" + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: db-secret + key: "db_password" + - name: MYSQL_PORT + value: "6033" + - name: NEXTCLOUD_TRUSTED_DOMAINS + value: "customer.drive.test.sunet.se" + - name: NEXTCLOUD_ADMIN_USER + value: admin + - name: NEXTCLOUD_VERSION_STRING + value: "25.0.3.3" + - name: NEXTCLOUD_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: nc-secret + key: "nc_admin_password" + - name: NEXTCLOUD_PASSWORDSALT + valueFrom: + secretKeyRef: + name: nc-secret + key: "nc_passwordsalt" + - name: NEXTCLOUD_INSTANCEID + valueFrom: + secretKeyRef: + name: nc-secret + key: "nc_instanceid" + - name: NEXTCLOUD_SECRET + valueFrom: + secretKeyRef: + name: nc-secret + key: "nc_secret" + - name: OBJECTSTORE_S3_REGION + value: "us-east-1" + - name: OBJECTSTORE_S3_HOST + value: "s3.sto4.safedc.net" + - name: OBJECTSTORE_S3_BUCKET + value: "primary-customer-drive-test.sunet.se" + - name: OBJECTSTORE_S3_KEY + valueFrom: + secretKeyRef: + name: s3-secret + key: "s3_key" + - name: OBJECTSTORE_S3_SECRET + valueFrom: + secretKeyRef: + name: s3-secret + key: "s3_secret" + - name: OBJECTSTORE_S3_USEPATH_STYLE + value: "true" + - name: OBJECTSTORE_S3_AUTOCREATE + value: "true" + - name: OBJECTSTORE_S3_SSL + value: "true" + - name: REDIS_HOST + value: "redis" + - name: SITE_NAME + value: "customer.drive.test.sunet.se" + command: ["/bin/bash", "-c", "/usr/bin/j2 -f env -o /var/www/html/config/config.php /tmp/config.php.template"] + volumes: + - name: nextcloud-config-template + configMap: + name: nextcloud-configmap + items: + - key: "config.php" + path: "config.php" + - name: default-config + configMap: + name: nextcloud-configmap + items: + - key: "000-default.conf" + path: "000-default.conf" + - name: hugepage + emptyDir: + medium: HugePages diff --git a/customers/base/nextcloud-ingress.yml b/customers/base/nextcloud-ingress.yml new file mode 100644 index 0000000..fb2548f --- /dev/null +++ b/customers/base/nextcloud-ingress.yml @@ -0,0 +1,31 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: customer-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + defaultBackend: + service: + name: customer-node + port: + number: 8443 + tls: + - hosts: + - customer.drive.test.sunet.se + secretName: tls-secret + + rules: + - host: customer.drive.test.sunet.se + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: customer-node + port: + number: 80 diff --git a/customers/base/nextcloud-service.yml b/customers/base/nextcloud-service.yml new file mode 100644 index 0000000..29d7fdd --- /dev/null +++ b/customers/base/nextcloud-service.yml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +items: +- apiVersion: v1 + kind: Service + metadata: + labels: + app: customer-node + name: customer-node + spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: customer-node + sessionAffinity: None + type: ClusterIP + status: + loadBalancer: {} +kind: List +metadata: + resourceVersion: "" + selfLink: "" diff --git a/customers/base/redis-deployment.yml b/customers/base/redis-deployment.yml new file mode 100644 index 0000000..c47da6b --- /dev/null +++ b/customers/base/redis-deployment.yml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + labels: + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: default + image: redis:latest + imagePullPolicy: Always + ports: + - containerPort: 6379 + args: + - "--save" + - "" + - "--appendonly" + - "no" + diff --git a/customers/base/redis-service.yml b/customers/base/redis-service.yml new file mode 100644 index 0000000..fedf42a --- /dev/null +++ b/customers/base/redis-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + app: redis + name: redis +spec: + ports: + - name: redis-port + nodePort: 30379 + port: 6379 + protocol: TCP + targetPort: 6379 + selector: + app: redis + type: NodePort diff --git a/customers/base/s3-service.yml b/customers/base/s3-service.yml new file mode 100644 index 0000000..842e1fa --- /dev/null +++ b/customers/base/s3-service.yml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: sto4 +spec: + type: ExternalName + externalName: s3.sto4.safedc.net + ports: + - port: 443 diff --git a/customers/overlays/vr/test/kustomization.yaml b/customers/overlays/vr/test/kustomization.yaml new file mode 100644 index 0000000..036c61d --- /dev/null +++ b/customers/overlays/vr/test/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: 'kustomize.config.k8s.io/v1beta1' +kind: Kustomization +bases: +- ../../../base +patchesStrategicMerge: + - nextcloud-deployment.yml + - nextcloud-ingress.yml diff --git a/customers/overlays/vr/test/nextcloud-deployment.yml b/customers/overlays/vr/test/nextcloud-deployment.yml new file mode 100644 index 0000000..861ceaa --- /dev/null +++ b/customers/overlays/vr/test/nextcloud-deployment.yml @@ -0,0 +1,34 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: customer-node + labels: + app: customer-node +spec: + template: + metadata: + labels: + app: customer-node + spec: + initContainers: + - image: docker.sunet.se/sunet/docker-jinja:latest + name: init-config + env: + - name: MYSQL_DATABASE + value: "nextcloud_vr" + - name: MYSQL_USER + value: "nextcloud_vr" + - name: GSS_MASTER_URL + value: "https://drive.test.sunet.se" + - name: LOOKUP_SERVER + value: "https://lookup.drive.test.sunet.se" + - name: MAIL_DOMAIN + value: "drive.test.sunet.se" + - name: MAIL_SMTPNAME + value: "noreply@drive.test.sunet.se" + - name: NEXTCLOUD_TRUSTED_DOMAINS + value: "vr.drive.test.sunet.se" + - name: OBJECTSTORE_S3_BUCKET + value: "primary-vr-drive-test.sunet.se" + - name: SITE_NAME + value: "vr.drive.test.sunet.se" diff --git a/customers/overlays/vr/test/nextcloud-ingress.yml b/customers/overlays/vr/test/nextcloud-ingress.yml new file mode 100644 index 0000000..9bec27e --- /dev/null +++ b/customers/overlays/vr/test/nextcloud-ingress.yml @@ -0,0 +1,26 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: customer-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + tls: + - hosts: + - vr.drive.test.sunet.se + secretName: tls-secret + + rules: + - host: vr.drive.test.sunet.se + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: customer-node + port: + number: 80 diff --git a/proxysql/base/kustomization.yaml b/proxysql/base/kustomization.yaml new file mode 100644 index 0000000..8fd990c --- /dev/null +++ b/proxysql/base/kustomization.yaml @@ -0,0 +1,6 @@ +resources: + - proxysql-configmap.yml + - proxysql-db-service.yml + - proxysql-deployment.yml + - proxysql-namespace.yml + - proxysql-service.yml diff --git a/proxysql/base/proxysql-configmap.yml b/proxysql/base/proxysql-configmap.yml new file mode 100644 index 0000000..24f81d8 --- /dev/null +++ b/proxysql/base/proxysql-configmap.yml @@ -0,0 +1,408 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: proxysql-configmap + namespace: proxysql +data: + proxysql.cnf: | + datadir="/var/lib/proxysql" + + admin_variables= + { + admin_credentials="admin:{{ADMIN_PASSWORD}};cluster:{{CLUSTER_PASSWORD}}" + mysql_ifaces="0.0.0.0:6032" + refresh_interval=2000 + cluster_username="cluster" + cluster_password="{{CLUSTER_PASSWORD}}" + } + + mysql_variables= + { + threads=4 + max_connections=2048 + default_query_delay=0 + default_query_timeout=36000000 + have_compress=true + poll_timeout=2000 + interfaces="0.0.0.0:6033;/tmp/proxysql.sock" + default_schema="information_schema" + stacksize=1048576 + server_version="8.0.23" + connect_timeout_server=3000 + monitor_username="proxysql" + monitor_password="{{MONITOR_PASSWORD}}" + monitor_history=600000 + monitor_connect_interval=60000 + monitor_ping_interval=10000 + monitor_read_only_interval=1500 + monitor_read_only_timeout=500 + ping_interval_server_msec=120000 + ping_timeout_server=500 + commands_stats=true + sessions_sort=true + connect_retries_on_failure=10 + } + + mysql_galera_hostgroups = + ( + { + + writer_hostgroup=10 + backup_writer_hostgroup=20 + reader_hostgroup=30 + offline_hostgroup=9999 + max_writers=1 + writer_is_also_reader=2 + max_transactions_behind=30 + active=1 + } + ) + mysql_query_rules = + ( + { + rule_id=100 + active=1 + match_pattern="^SELECT .* FOR UPDATE" + destination_hostgroup=10 + apply=1 + }, + { + rule_id=200 + active=1 + match_pattern="^SELECT .*" + destination_hostgroup=30 + apply=1 + }, + { + rule_id=300 + active=1 + match_pattern=".*" + destination_hostgroup=10 + apply=1 + } + ) + mysql_servers = + ( + { address="multinode-db1" , port=3306 , hostgroup=10, max_connections=100 }, + { address="multinode-db2" , port=3306 , hostgroup=20, max_connections=100 }, + { address="multinode-db3" , port=3306 , hostgroup=30, max_connections=100 } + ) + + mysql_users = + ( + { + username="nextcloud_antagning" + password="{{ANTAGNING_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_bth" + password="{{BTH_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_chalmers" + password="{{CHALMERS_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_du" + password="{{DU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_esh" + password="{{ESH_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_fhs" + password="{{FHS_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_gu" + password="{{GU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_hb" + password="{{HB_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_hhs" + password="{{HHS_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_hig" + password="{{HIG_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_his" + password="{{HIS_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_hj" + password="{{HJ_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_hv" + password="{{HV_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_irf" + password="{{IRF_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_kb" + password="{{KB_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_ki" + password="{{KI_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_kkh" + password="{{KKH_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_kmh" + password="{{KMH_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_konstfack" + password="{{KONSTFACK_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_kth" + password="{{KTH_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_kva" + password="{{KVA_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_liu" + password="{{LIU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_lnu" + password="{{LNU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_ltu" + password="{{LTU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_lu" + password="{{LU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_mau" + password="{{MAU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_miun" + password="{{MIUN_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_nordunet" + password="{{NORDUNET_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_nrm" + password="{{NRM_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_oru" + password="{{ORU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_rkh" + password="{{RKH_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_shh" + password="{{SHH_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_sics" + password="{{SICS_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_slu" + password="{{SLU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_smhi" + password="{{SMHI_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_sp" + password="{{SP_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_swamid" + password="{{SWAMID_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_ths" + password="{{THS_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_uhr" + password="{{UHR_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_umu" + password="{{UMU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_uniarts" + password="{{UNIARTS_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_uu" + password="{{UU_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_vinnova" + password="{{VINNOVA_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + { + username="nextcloud_vr" + password="{{VR_PASSWORD}}" + default_hostgroup=10 + transaction_persistent=1 + active=1 + }, + ) + + proxysql_servers = + ( + { hostname = "proxysql-0.proxysqlcluster", port = 6032, weight = 1 }, + { hostname = "proxysql-1.proxysqlcluster", port = 6032, weight = 1 }, + { hostname = "proxysql-2.proxysqlcluster", port = 6032, weight = 1 } + ) diff --git a/proxysql/base/proxysql-db-service.yml b/proxysql/base/proxysql-db-service.yml new file mode 100644 index 0000000..e83f156 --- /dev/null +++ b/proxysql/base/proxysql-db-service.yml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db1 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db1.drive.test.sunet.se + ports: + - port: 3306 +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db2 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db2.drive.test.sunet.se + ports: + - port: 3306 +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db3 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db3.drive.test.sunet.se + ports: + - port: 3306 diff --git a/proxysql/base/proxysql-deployment.yml b/proxysql/base/proxysql-deployment.yml new file mode 100644 index 0000000..2efcda4 --- /dev/null +++ b/proxysql/base/proxysql-deployment.yml @@ -0,0 +1,311 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: proxysql + namespace: proxysql + labels: + app: proxysql +spec: + replicas: 3 + serviceName: proxysqlcluster + selector: + matchLabels: + app: proxysql + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - metadata: + name: proxysql-data + namespace: proxysql + spec: + storageClassName: mayastor-3 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + - metadata: + name: proxysql-etc + namespace: proxysql + spec: + storageClassName: mayastor-3 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + template: + metadata: + labels: + app: proxysql + spec: + restartPolicy: Always + containers: + - image: docker.sunet.se/drive/proxysql:latest + name: proxysql + volumeMounts: + - name: proxysql-data + mountPath: /var/lib/proxysql + subPath: data + - name: proxysql-etc + mountPath: /etc + subPath: etc + ports: + - containerPort: 6033 + name: proxysql-mysql + - containerPort: 6032 + name: proxysql-admin + initContainers: + - image: docker.sunet.se/sunet/docker-jinja:latest + name: init-config + volumeMounts: + - name: proxysql-config-template + mountPath: /tmp/proxysql.cnf.template + subPath: proxysql.cnf + - name: proxysql-etc + mountPath: /etc + subPath: etc + env: + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_admin_password" + - name: CLUSTER_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_cluster_password" + - name: MONITOR_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_monitor_password" + - name: ANTAGNING_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_antagning_password" + - name: BTH_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_bth_password" + - name: CHALMERS_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_chalmers_password" + - name: DU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_du_password" + - name: ESH_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_esh_password" + - name: FHS_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_fhs_password" + - name: GU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_gu_password" + - name: HB_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_hb_password" + - name: HHS_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_hhs_password" + - name: HIG_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_hig_password" + - name: HIS_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_his_password" + - name: HJ_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_hj_password" + - name: HV_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_hv_password" + - name: IRF_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_irf_password" + - name: KB_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_kb_password" + - name: KI_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_ki_password" + - name: KKH_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_kkh_password" + - name: KMH_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_kmh_password" + - name: KONSTFACK_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_konstfack_password" + - name: KTH_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_kth_password" + - name: KVA_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_kva_password" + - name: LIU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_liu_password" + - name: LNU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_lnu_password" + - name: LTU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_ltu_password" + - name: LU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_lu_password" + - name: MAU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_mau_password" + - name: MIUN_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_miun_password" + - name: NORDUNET_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_nordunet_password" + - name: NRM_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_nrm_password" + - name: ORU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_oru_password" + - name: RKH_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_rkh_password" + - name: SHH_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_shh_password" + - name: SICS_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_sics_password" + - name: SLU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_slu_password" + - name: SMHI_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_smhi_password" + - name: SP_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_sp_password" + - name: SWAMID_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_swamid_password" + - name: THS_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_ths_password" + - name: UHR_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_uhr_password" + - name: UMU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_umu_password" + - name: UNIARTS_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_uniarts_password" + - name: UU_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_uu_password" + - name: VINNOVA_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_vinnova_password" + - name: VR_PASSWORD + valueFrom: + secretKeyRef: + name: proxysql-secret + key: "proxysql_vr_password" + command: ["/bin/sh", "-c", "/usr/bin/j2 -f env -o /etc/proxysql.cnf /tmp/proxysql.cnf.template"] + volumes: + - name: proxysql-config-template + configMap: + name: proxysql-configmap + items: + - key: "proxysql.cnf" + path: "proxysql.cnf" diff --git a/proxysql/base/proxysql-namespace.yml b/proxysql/base/proxysql-namespace.yml new file mode 100644 index 0000000..f418b55 --- /dev/null +++ b/proxysql/base/proxysql-namespace.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: proxysql +spec: + finalizers: + - kubernetes diff --git a/proxysql/base/proxysql-service.yml b/proxysql/base/proxysql-service.yml new file mode 100644 index 0000000..96d6bcd --- /dev/null +++ b/proxysql/base/proxysql-service.yml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + app: proxysql + name: proxysql + namespace: proxysql +spec: + ports: + - name: proxysql-mysql + nodePort: 30033 + port: 6033 + protocol: TCP + targetPort: 6033 + - name: proxysql-admin + nodePort: 30032 + port: 6032 + protocol: TCP + targetPort: 6032 + selector: + app: proxysql + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: proxysqlcluster + namespace: proxysql + labels: + app: proxysql +spec: + clusterIP: None + ports: + - port: 6032 + name: proxysql-admin + selector: + app: proxysql diff --git a/proxysql/overlays/prod/kustomization.yaml b/proxysql/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..27ecbfb --- /dev/null +++ b/proxysql/overlays/prod/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: 'kustomize.config.k8s.io/v1beta1' +kind: Kustomization +bases: +- ../../base +patchesStrategicMerge: + - proxysql-db-service.yml diff --git a/proxysql/overlays/prod/proxysql-db-service.yml b/proxysql/overlays/prod/proxysql-db-service.yml new file mode 100644 index 0000000..e83f156 --- /dev/null +++ b/proxysql/overlays/prod/proxysql-db-service.yml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db1 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db1.drive.test.sunet.se + ports: + - port: 3306 +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db2 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db2.drive.test.sunet.se + ports: + - port: 3306 +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db3 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db3.drive.test.sunet.se + ports: + - port: 3306 diff --git a/proxysql/overlays/test/kustomization.yaml b/proxysql/overlays/test/kustomization.yaml new file mode 100644 index 0000000..27ecbfb --- /dev/null +++ b/proxysql/overlays/test/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: 'kustomize.config.k8s.io/v1beta1' +kind: Kustomization +bases: +- ../../base +patchesStrategicMerge: + - proxysql-db-service.yml diff --git a/proxysql/overlays/test/proxysql-db-service.yml b/proxysql/overlays/test/proxysql-db-service.yml new file mode 100644 index 0000000..e83f156 --- /dev/null +++ b/proxysql/overlays/test/proxysql-db-service.yml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db1 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db1.drive.test.sunet.se + ports: + - port: 3306 +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db2 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db2.drive.test.sunet.se + ports: + - port: 3306 +--- +apiVersion: v1 +kind: Service +metadata: + name: multinode-db3 + namespace: proxysql +spec: + type: ExternalName + externalName: multinode-db3.drive.test.sunet.se + ports: + - port: 3306