From 32e4a99cef0112cb7054f92e4cb0006735c550eb Mon Sep 17 00:00:00 2001 From: Patrik Lundin Date: Thu, 3 Oct 2024 20:12:59 +0200 Subject: [PATCH] Add initial forgejo runner config --- global/overlay/etc/puppet/cosmos-rules.yaml | 4 + .../puppet/modules/cdn/manifests/runner.pp | 46 + .../templates/runner/docker-compose.yml.erb | 63 ++ .../cdn/templates/runner/seccomp.json.erb | 841 ++++++++++++++++++ .../overlay/etc/hiera/data/local.eyaml | 2 + 5 files changed, 956 insertions(+) create mode 100644 global/overlay/etc/puppet/modules/cdn/manifests/runner.pp create mode 100644 global/overlay/etc/puppet/modules/cdn/templates/runner/docker-compose.yml.erb create mode 100644 global/overlay/etc/puppet/modules/cdn/templates/runner/seccomp.json.erb create mode 100644 internal-sto3-test-runner-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml diff --git a/global/overlay/etc/puppet/cosmos-rules.yaml b/global/overlay/etc/puppet/cosmos-rules.yaml index e1b1912..baf2ed7 100644 --- a/global/overlay/etc/puppet/cosmos-rules.yaml +++ b/global/overlay/etc/puppet/cosmos-rules.yaml @@ -11,3 +11,7 @@ '^cdn-prod-l4lb-[0-9]+.sunet.se$': sunet::dockerhost2: cdn::l4lb: + +'^internal-.+-test-runner-[0-9]+.\.cdn\.sunet\.se$': + sunet::dockerhost2: + cdn::runner: diff --git a/global/overlay/etc/puppet/modules/cdn/manifests/runner.pp b/global/overlay/etc/puppet/modules/cdn/manifests/runner.pp new file mode 100644 index 0000000..0486e49 --- /dev/null +++ b/global/overlay/etc/puppet/modules/cdn/manifests/runner.pp @@ -0,0 +1,46 @@ +# Configure the SUNET CDN forgejo runner +class cdn::runner( +) +{ + $runner_token = lookup({ 'name' => 'runner_token', 'default_value' => undef }) + + if $runner_token { + + file { '/opt/forgejo-runner': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0750', + } + + file { '/opt/forgejo-runner/data': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0750', + } + + file { '/opt/forgejo-runner/data/seccomp.json': + ensure => file, + owner => 'root', + group => 'root', + mode => '0640', + content => template('cdn/runner/seccomp.json.erb'), + } + + file { '/opt/forgejo-runner/docker_certs': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0750', + } + + sunet::docker_compose { 'sunet-cdn-runner': + content => template('cdn/runner/docker-compose.yml.erb'), + service_name => 'cdn-runner', + compose_dir => '/opt/sunet-cdn/compose/runner', + compose_filename => 'docker-compose.yml', + description => 'SUNET CDN forgejo runner', + } + } +} diff --git a/global/overlay/etc/puppet/modules/cdn/templates/runner/docker-compose.yml.erb b/global/overlay/etc/puppet/modules/cdn/templates/runner/docker-compose.yml.erb new file mode 100644 index 0000000..59b0262 --- /dev/null +++ b/global/overlay/etc/puppet/modules/cdn/templates/runner/docker-compose.yml.erb @@ -0,0 +1,63 @@ +version: '3.8' + +# Based on combination of https://forgejo.org/docs/latest/admin/actions/ and +# https://code.forgejo.org/forgejo/runner/src/branch/main/examples/docker-compose/compose-forgejo-and-runner.yml +# +# Also configures a custom seccomp profile in runner config because the slash +# storage engine uses io_uring which the default profile does no allow, and +# this makes code tests fail during the build. + +services: + docker-in-docker: + image: docker:dind + hostname: docker # Must set hostname as TLS certificates are only valid for docker or localhost + privileged: 'true' + environment: + DOCKER_TLS_CERTDIR: /certs + DOCKER_HOST: docker-in-docker + volumes: + - /opt/forgejo-runner/docker_certs:/certs + + runner-register: + image: 'code.forgejo.org/forgejo/runner:3.5.0' + depends_on: + docker-in-docker: + condition: service_started + # User without root privileges, but with access to `./data`. + user: 1001:1001 + volumes: + - /opt/forgejo-runner/data:/data + command: >- + bash -ec ' + while : ; do + if [ -f .runner ]; then echo "runner already registered, exiting"; exit; fi + forgejo-runner register --no-interactive --name <%= @networking.fqdn %> --instance https://platform.sunet.se --token <%= @runner_token %> --labels docker:docker://node:20-bookworm,ubuntu-22.04:docker://catthehacker/ubuntu:act-22.04 && break; + sleep 1 ; + done ; + forgejo-runner generate-config > config.yml ; + sed -i -e "s|network: .*|network: host|" config.yml ; + sed -i -e "s|^ envs:$$| envs:\n DOCKER_HOST: tcp://docker:2376\n DOCKER_TLS_VERIFY: 1\n DOCKER_CERT_PATH: /certs/client|" config.yml ; + sed -i -e "s|^ options:| options: -v /certs/client:/certs/client --security-opt seccomp=/data/seccomp.json|" config.yml ; + sed -i -e "s| valid_volumes: \[\]$$| valid_volumes:\n - /certs/client|" config.yml ; + ' + + runner-daemon: + image: code.forgejo.org/forgejo/runner:3.5.0 + user: 1001:1001 + links: + - docker-in-docker + depends_on: + runner-register: + condition: service_completed_successfully + environment: + DOCKER_HOST: tcp://docker:2376 + DOCKER_CERT_PATH: /certs/client + DOCKER_TLS_VERIFY: "1" + volumes: + - /opt/forgejo-runner/data:/data + - /opt/forgejo-runner/docker_certs:/certs + command: + - 'forgejo-runner' + - '--config' + - 'config.yml' + - 'daemon' diff --git a/global/overlay/etc/puppet/modules/cdn/templates/runner/seccomp.json.erb b/global/overlay/etc/puppet/modules/cdn/templates/runner/seccomp.json.erb new file mode 100644 index 0000000..c6b6d66 --- /dev/null +++ b/global/overlay/etc/puppet/modules/cdn/templates/runner/seccomp.json.erb @@ -0,0 +1,841 @@ +{ + "defaultAction": "SCMP_ACT_ERRNO", + "defaultErrnoRet": 1, + "archMap": [ + { + "architecture": "SCMP_ARCH_X86_64", + "subArchitectures": [ + "SCMP_ARCH_X86", + "SCMP_ARCH_X32" + ] + }, + { + "architecture": "SCMP_ARCH_AARCH64", + "subArchitectures": [ + "SCMP_ARCH_ARM" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64" + ] + }, + { + "architecture": "SCMP_ARCH_S390X", + "subArchitectures": [ + "SCMP_ARCH_S390" + ] + }, + { + "architecture": "SCMP_ARCH_RISCV64", + "subArchitectures": null + } + ], + "syscalls": [ + { + "names": [ + "accept", + "accept4", + "access", + "adjtimex", + "alarm", + "bind", + "brk", + "cachestat", + "capget", + "capset", + "chdir", + "chmod", + "chown", + "chown32", + "clock_adjtime", + "clock_adjtime64", + "clock_getres", + "clock_getres_time64", + "clock_gettime", + "clock_gettime64", + "clock_nanosleep", + "clock_nanosleep_time64", + "close", + "close_range", + "connect", + "copy_file_range", + "creat", + "dup", + "dup2", + "dup3", + "epoll_create", + "epoll_create1", + "epoll_ctl", + "epoll_ctl_old", + "epoll_pwait", + "epoll_pwait2", + "epoll_wait", + "epoll_wait_old", + "eventfd", + "eventfd2", + "execve", + "execveat", + "exit", + "exit_group", + "faccessat", + "faccessat2", + "fadvise64", + "fadvise64_64", + "fallocate", + "fanotify_mark", + "fchdir", + "fchmod", + "fchmodat", + "fchmodat2", + "fchown", + "fchown32", + "fchownat", + "fcntl", + "fcntl64", + "fdatasync", + "fgetxattr", + "flistxattr", + "flock", + "fork", + "fremovexattr", + "fsetxattr", + "fstat", + "fstat64", + "fstatat64", + "fstatfs", + "fstatfs64", + "fsync", + "ftruncate", + "ftruncate64", + "futex", + "futex_requeue", + "futex_time64", + "futex_wait", + "futex_waitv", + "futex_wake", + "futimesat", + "getcpu", + "getcwd", + "getdents", + "getdents64", + "getegid", + "getegid32", + "geteuid", + "geteuid32", + "getgid", + "getgid32", + "getgroups", + "getgroups32", + "getitimer", + "getpeername", + "getpgid", + "getpgrp", + "getpid", + "getppid", + "getpriority", + "getrandom", + "getresgid", + "getresgid32", + "getresuid", + "getresuid32", + "getrlimit", + "get_robust_list", + "getrusage", + "getsid", + "getsockname", + "getsockopt", + "get_thread_area", + "gettid", + "gettimeofday", + "getuid", + "getuid32", + "getxattr", + "inotify_add_watch", + "inotify_init", + "inotify_init1", + "inotify_rm_watch", + "io_cancel", + "ioctl", + "io_destroy", + "io_getevents", + "io_pgetevents", + "io_pgetevents_time64", + "ioprio_get", + "ioprio_set", + "io_setup", + "io_submit", + "ipc", + "kill", + "landlock_add_rule", + "landlock_create_ruleset", + "landlock_restrict_self", + "lchown", + "lchown32", + "lgetxattr", + "link", + "linkat", + "listen", + "listxattr", + "llistxattr", + "_llseek", + "lremovexattr", + "lseek", + "lsetxattr", + "lstat", + "lstat64", + "madvise", + "map_shadow_stack", + "membarrier", + "memfd_create", + "memfd_secret", + "mincore", + "mkdir", + "mkdirat", + "mknod", + "mknodat", + "mlock", + "mlock2", + "mlockall", + "mmap", + "mmap2", + "mprotect", + "mq_getsetattr", + "mq_notify", + "mq_open", + "mq_timedreceive", + "mq_timedreceive_time64", + "mq_timedsend", + "mq_timedsend_time64", + "mq_unlink", + "mremap", + "msgctl", + "msgget", + "msgrcv", + "msgsnd", + "msync", + "munlock", + "munlockall", + "munmap", + "name_to_handle_at", + "nanosleep", + "newfstatat", + "_newselect", + "open", + "openat", + "openat2", + "pause", + "pidfd_open", + "pidfd_send_signal", + "pipe", + "pipe2", + "pkey_alloc", + "pkey_free", + "pkey_mprotect", + "poll", + "ppoll", + "ppoll_time64", + "prctl", + "pread64", + "preadv", + "preadv2", + "prlimit64", + "process_mrelease", + "pselect6", + "pselect6_time64", + "pwrite64", + "pwritev", + "pwritev2", + "read", + "readahead", + "readlink", + "readlinkat", + "readv", + "recv", + "recvfrom", + "recvmmsg", + "recvmmsg_time64", + "recvmsg", + "remap_file_pages", + "removexattr", + "rename", + "renameat", + "renameat2", + "restart_syscall", + "rmdir", + "rseq", + "rt_sigaction", + "rt_sigpending", + "rt_sigprocmask", + "rt_sigqueueinfo", + "rt_sigreturn", + "rt_sigsuspend", + "rt_sigtimedwait", + "rt_sigtimedwait_time64", + "rt_tgsigqueueinfo", + "sched_getaffinity", + "sched_getattr", + "sched_getparam", + "sched_get_priority_max", + "sched_get_priority_min", + "sched_getscheduler", + "sched_rr_get_interval", + "sched_rr_get_interval_time64", + "sched_setaffinity", + "sched_setattr", + "sched_setparam", + "sched_setscheduler", + "sched_yield", + "seccomp", + "select", + "semctl", + "semget", + "semop", + "semtimedop", + "semtimedop_time64", + "send", + "sendfile", + "sendfile64", + "sendmmsg", + "sendmsg", + "sendto", + "setfsgid", + "setfsgid32", + "setfsuid", + "setfsuid32", + "setgid", + "setgid32", + "setgroups", + "setgroups32", + "setitimer", + "setpgid", + "setpriority", + "setregid", + "setregid32", + "setresgid", + "setresgid32", + "setresuid", + "setresuid32", + "setreuid", + "setreuid32", + "setrlimit", + "set_robust_list", + "setsid", + "setsockopt", + "set_thread_area", + "set_tid_address", + "setuid", + "setuid32", + "setxattr", + "shmat", + "shmctl", + "shmdt", + "shmget", + "shutdown", + "sigaltstack", + "signalfd", + "signalfd4", + "sigprocmask", + "sigreturn", + "socketcall", + "socketpair", + "splice", + "stat", + "stat64", + "statfs", + "statfs64", + "statx", + "symlink", + "symlinkat", + "sync", + "sync_file_range", + "syncfs", + "sysinfo", + "tee", + "tgkill", + "time", + "timer_create", + "timer_delete", + "timer_getoverrun", + "timer_gettime", + "timer_gettime64", + "timer_settime", + "timer_settime64", + "timerfd_create", + "timerfd_gettime", + "timerfd_gettime64", + "timerfd_settime", + "timerfd_settime64", + "times", + "tkill", + "truncate", + "truncate64", + "ugetrlimit", + "umask", + "uname", + "unlink", + "unlinkat", + "utime", + "utimensat", + "utimensat_time64", + "utimes", + "vfork", + "vmsplice", + "wait4", + "waitid", + "waitpid", + "write", + "writev" + ], + "action": "SCMP_ACT_ALLOW" + }, + { + "names": [ + "process_vm_readv", + "process_vm_writev", + "ptrace" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "minKernel": "4.8" + } + }, + { + "names": [ + "socket" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 40, + "op": "SCMP_CMP_NE" + } + ] + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 0, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 8, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 131072, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 131080, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": [ + "personality" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 4294967295, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": [ + "sync_file_range2", + "swapcontext" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": [ + "ppc64le" + ] + } + }, + { + "names": [ + "arm_fadvise64_64", + "arm_sync_file_range", + "sync_file_range2", + "breakpoint", + "cacheflush", + "set_tls" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": [ + "arm", + "arm64" + ] + } + }, + { + "names": [ + "arch_prctl" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": [ + "amd64", + "x32" + ] + } + }, + { + "names": [ + "modify_ldt" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": [ + "amd64", + "x32", + "x86" + ] + } + }, + { + "names": [ + "s390_pci_mmio_read", + "s390_pci_mmio_write", + "s390_runtime_instr" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": [ + "s390", + "s390x" + ] + } + }, + { + "names": [ + "riscv_flush_icache" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": [ + "riscv64" + ] + } + }, + { + "names": [ + "open_by_handle_at" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_DAC_READ_SEARCH" + ] + } + }, + { + "names": [ + "bpf", + "clone", + "clone3", + "fanotify_init", + "fsconfig", + "fsmount", + "fsopen", + "fspick", + "lookup_dcookie", + "mount", + "mount_setattr", + "move_mount", + "open_tree", + "perf_event_open", + "quotactl", + "quotactl_fd", + "setdomainname", + "sethostname", + "setns", + "syslog", + "umount", + "umount2", + "unshare" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + } + }, + { + "names": [ + "clone" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 2114060288, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ], + "arches": [ + "s390", + "s390x" + ] + } + }, + { + "names": [ + "clone" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 1, + "value": 2114060288, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "comment": "s390 parameter ordering for clone is different", + "includes": { + "arches": [ + "s390", + "s390x" + ] + }, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + } + }, + { + "names": [ + "clone3" + ], + "action": "SCMP_ACT_ERRNO", + "errnoRet": 38, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + } + }, + { + "names": [ + "reboot" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_BOOT" + ] + } + }, + { + "names": [ + "chroot" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_CHROOT" + ] + } + }, + { + "names": [ + "delete_module", + "init_module", + "finit_module" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_MODULE" + ] + } + }, + { + "names": [ + "acct" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_PACCT" + ] + } + }, + { + "names": [ + "kcmp", + "pidfd_getfd", + "process_madvise", + "process_vm_readv", + "process_vm_writev", + "ptrace" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_PTRACE" + ] + } + }, + { + "names": [ + "iopl", + "ioperm" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_RAWIO" + ] + } + }, + { + "names": [ + "settimeofday", + "stime", + "clock_settime", + "clock_settime64" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_TIME" + ] + } + }, + { + "names": [ + "vhangup" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_TTY_CONFIG" + ] + } + }, + { + "names": [ + "get_mempolicy", + "mbind", + "set_mempolicy", + "set_mempolicy_home_node" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYS_NICE" + ] + } + }, + { + "names": [ + "syslog" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_SYSLOG" + ] + } + }, + { + "names": [ + "bpf" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_BPF" + ] + } + }, + { + "names": [ + "perf_event_open" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": [ + "CAP_PERFMON" + ] + } + }, + { + "names": [ + "io_uring_enter", + "io_uring_register", + "io_uring_setup" + ], + "action": "SCMP_ACT_ALLOW" + } + ] +} diff --git a/internal-sto3-test-runner-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml b/internal-sto3-test-runner-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml new file mode 100644 index 0000000..dff336d --- /dev/null +++ b/internal-sto3-test-runner-1.cdn.sunet.se/overlay/etc/hiera/data/local.eyaml @@ -0,0 +1,2 @@ +--- +runner_token: ENC[PKCS7,MIIC7AYJKoZIhvcNAQcDoIIC3TCCAtkCAQAxggKUMIICkAIBADB4MGAxCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxMTAvBgNVBAMMKGludGVybmFsLXN0bzMtdGVzdC1ydW5uZXItMS5jZG4uc3VuZXQuc2UCFB8mr7Xs+11gtQ/ESDzi1+8CDVYHMA0GCSqGSIb3DQEBAQUABIICAMOCfrRasdX7b5ch7JEmc4Pb//CVwfr9eWlwnlWE4UUr4lu6WmIHj1QuTjETpcw9G4im6hGKjV+ygUN7WPRzTwxrT5qnsXj/AYM0pI4lfAfYzkejUKUDjv6kVbwfeyZpv2E6Od8OkpyqEWM4deepj2z+gUR4UTCsHsaH6rkB4+fM4YryKcV4XibUfuTwqNvoVVLXhrajwIvzuFAorGAVm0UzYUKmfuZRNzd0pgX20eKiHxEyoQkmdfFybVMO6KGmTMgTo5IXeyxCwU0F+D6LmVkr3mdAuy3XnzWusouGOL+wiCOO5f6lM6SoC7vcJryQYIyaevYIg9FhE9Y/4ZKNTUNosWL/z+zjJ7FjE+3NZOPepga5gIXV0Nngxz3ZXluhELxM70StGuikeEfAOW9XB8ldtpg36yZtkgHNIeCPCPwZpZhNJanTxj6DinFNcqaiKG0Bv928bTPq9SUzpMoFDX2HVFepId1dZZwlUn2zU9mG1NJoHS0WdX1Ex/K6lM9GgZhlAXARrfIhw4acejObU4pNnmbW7RIqI+vhdTW3kNF9RukfcrqjPPm/8QjrfOx5d51hQByUDj4NuXFt2RwX9z5V7LqQmqyVfph4pazO2EBxRSyxSn+mUgkPQeZ2pjKz2X5W2rV+bp3oLg8txe8ssewVN8Q07AzDs0m65Vhur5OCMDwGCSqGSIb3DQEHATAdBglghkgBZQMEASoEEBm7Xl7JL1MRLgIp/wHQh4CAELL4oYGSJe4mCqdYXxyIr8g=]