From 1122c3f0e21dfc99e7aee7614eb8a382b9bd0f65 Mon Sep 17 00:00:00 2001
From: Felix Wolf <felix.wolf@blueworld-gmbh.de>
Date: Mon, 6 Apr 2026 15:24:14 +0200
Subject: [PATCH] feat: Implement S3 to Storage Box backup

Introduces a daily Kubernetes CronJob that utilizes rclone to perform compressed backups of oCIS S3 data to a Hetzner Storage Box via SFTP.

This new backup mechanism requires the manual creation of an 'ocis-storagebox-credentials' secret, which holds the Storage Box host, user, and SSH private key. A check is added to the secret initialization job to ensure this essential external secret exists.
---
 CLAUDE.md                                     |   1 +
 .../ocis/ytt/s3-backup-cronjob.ytt.yaml       | 109 ++++++++++++++++++
 prototypes/ocis/ytt/s3-secret-job.ytt.yaml    |   5 +
 .../ocis/configmap-auth-service.yaml          |   2 +-
 .../envs/production/ocis/configmap-graph.yaml |   2 +-
 .../ocis/configmap-storage-users.yaml         |   2 +-
 .../ocis/cronjob-ocis-s3-backup.yaml          |  99 ++++++++++++++++
 .../production/ocis/job-ocis-secret-init.yaml |   5 +
 .../ocis/serviceaccount-ocis-s3-backup.yaml   |   7 ++
 9 files changed, 229 insertions(+), 3 deletions(-)
 create mode 100644 prototypes/ocis/ytt/s3-backup-cronjob.ytt.yaml
 create mode 100644 rendered/envs/production/ocis/cronjob-ocis-s3-backup.yaml
 create mode 100644 rendered/envs/production/ocis/serviceaccount-ocis-s3-backup.yaml

diff --git a/CLAUDE.md b/CLAUDE.md
index 72cb37d..af634d7 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -81,4 +81,5 @@ kubectl apply -f rendered/envs/production/<app>/ --server-side  # Deploy
 - When adding a new application that uses a Helm chart generating secrets, configure all `secretRefs` to point to pre-created secret names and use an init Job to generate them.
 - Known external secrets (not in git, created manually):
   - `ocis/ocis-s3-credentials` — Hetzner S3 access key and secret key
+  - `ocis/ocis-storagebox-credentials` — Hetzner Storage Box host, user, and SSH private key (for S3 backup to Helsinki)
   - `cert-manager/letsencrypt-account-key` — ACME account key (auto-generated by cert-manager)
diff --git a/prototypes/ocis/ytt/s3-backup-cronjob.ytt.yaml b/prototypes/ocis/ytt/s3-backup-cronjob.ytt.yaml
new file mode 100644
index 0000000..329d2e0
--- /dev/null
+++ b/prototypes/ocis/ytt/s3-backup-cronjob.ytt.yaml
@@ -0,0 +1,109 @@
+#@ load("@ytt:data", "data")
+
+#@ ns = data.values.application.namespace
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ocis-s3-backup
+  namespace: #@ ns
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: ocis-s3-backup
+  namespace: #@ ns
+spec:
+  schedule: "0 2 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 86400
+      template:
+        spec:
+          restartPolicy: OnFailure
+          serviceAccountName: ocis-s3-backup
+          containers:
+            - name: backup
+              image: alpine:3.20
+              resources:
+                requests:
+                  memory: 128Mi
+                  cpu: 50m
+              command:
+                - sh
+                - -c
+                - |
+                  set -e
+                  apk add --no-cache rclone >/dev/null 2>&1
+
+                  mkdir -p /tmp/rclone
+                  cat > /tmp/rclone/rclone.conf <<CONF
+                  [s3]
+                  type = s3
+                  provider = Other
+                  access_key_id = ${S3_ACCESS_KEY}
+                  secret_access_key = ${S3_SECRET_KEY}
+                  endpoint = https://nbg1.your-objectstorage.com
+                  acl = private
+
+                  [storagebox]
+                  type = sftp
+                  host = ${STORAGEBOX_HOST}
+                  port = 23
+                  user = ${STORAGEBOX_USER}
+                  key_file = /etc/storagebox/ssh-key
+                  shell_type = none
+                  md5sum_command = none
+                  sha1sum_command = none
+
+                  [backup]
+                  type = compress
+                  remote = storagebox:ocis-backup
+                  CONF
+
+                  echo "Syncing S3 bucket to Storage Box (compressed)..."
+                  rclone sync s3:ocis-tr1ceracop backup: \
+                    --config /tmp/rclone/rclone.conf \
+                    --transfers 4 \
+                    -v
+
+                  rm -rf /tmp/rclone
+                  echo "Backup complete."
+              env:
+                - name: S3_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: ocis-s3-credentials
+                      key: accessKey
+                - name: S3_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: ocis-s3-credentials
+                      key: secretKey
+                - name: STORAGEBOX_HOST
+                  valueFrom:
+                    secretKeyRef:
+                      name: ocis-storagebox-credentials
+                      key: host
+                - name: STORAGEBOX_USER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ocis-storagebox-credentials
+                      key: user
+              volumeMounts:
+                - name: storagebox-ssh
+                  mountPath: /etc/storagebox
+                  readOnly: true
+          volumes:
+            - name: storagebox-ssh
+              secret:
+                secretName: ocis-storagebox-credentials
+                items:
+                  - key: ssh-key
+                    path: ssh-key
+                defaultMode: 0400
diff --git a/prototypes/ocis/ytt/s3-secret-job.ytt.yaml b/prototypes/ocis/ytt/s3-secret-job.ytt.yaml
index 02863a5..f3113d2 100644
--- a/prototypes/ocis/ytt/s3-secret-job.ytt.yaml
+++ b/prototypes/ocis/ytt/s3-secret-job.ytt.yaml
@@ -96,6 +96,11 @@ spec:
                 exit 1
               fi
 
+              if ! kubectl get secret ocis-storagebox-credentials -n "${NAMESPACE}" >/dev/null 2>&1; then
+                echo "ERROR: External secret ocis-storagebox-credentials must be created manually"
+                exit 1
+              fi
+
               # Admin user
               create_secret_if_missing ocis-admin-user \
                 --from-literal=password="$(gen_random 32)" \
diff --git a/rendered/envs/production/ocis/configmap-auth-service.yaml b/rendered/envs/production/ocis/configmap-auth-service.yaml
index 7dd6059..bd50c45 100644
--- a/rendered/envs/production/ocis/configmap-auth-service.yaml
+++ b/rendered/envs/production/ocis/configmap-auth-service.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 data:
-  service-account-id: 1a1c862d-11c0-4c04-b078-c20277e455f5
+  service-account-id: 2387fad3-be34-4b10-948b-421873985560
 kind: ConfigMap
 metadata:
   annotations:
diff --git a/rendered/envs/production/ocis/configmap-graph.yaml b/rendered/envs/production/ocis/configmap-graph.yaml
index 438ab3b..3be71db 100644
--- a/rendered/envs/production/ocis/configmap-graph.yaml
+++ b/rendered/envs/production/ocis/configmap-graph.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 data:
-  application-id: 802e4fae-4e57-40ff-8c3e-fd92717b2ac3
+  application-id: d019e54c-51c8-46ab-aded-87182aafcee4
 kind: ConfigMap
 metadata:
   annotations:
diff --git a/rendered/envs/production/ocis/configmap-storage-users.yaml b/rendered/envs/production/ocis/configmap-storage-users.yaml
index 94de0cb..43766b5 100644
--- a/rendered/envs/production/ocis/configmap-storage-users.yaml
+++ b/rendered/envs/production/ocis/configmap-storage-users.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 data:
-  storage-uuid: a8925d16-9aea-4c77-923a-49bb7d8eeb77
+  storage-uuid: 30a27136-b87a-431f-9d0d-0cfec28061e4
 kind: ConfigMap
 metadata:
   annotations:
diff --git a/rendered/envs/production/ocis/cronjob-ocis-s3-backup.yaml b/rendered/envs/production/ocis/cronjob-ocis-s3-backup.yaml
new file mode 100644
index 0000000..21543fb
--- /dev/null
+++ b/rendered/envs/production/ocis/cronjob-ocis-s3-backup.yaml
@@ -0,0 +1,99 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  annotations:
+    a8r.io/repository: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
+  name: ocis-s3-backup
+  namespace: ocis
+spec:
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - command:
+                - sh
+                - -c
+                - |
+                  set -e
+                  apk add --no-cache rclone >/dev/null 2>&1
+
+                  mkdir -p /tmp/rclone
+                  cat > /tmp/rclone/rclone.conf <<CONF
+                  [s3]
+                  type = s3
+                  provider = Other
+                  access_key_id = ${S3_ACCESS_KEY}
+                  secret_access_key = ${S3_SECRET_KEY}
+                  endpoint = https://nbg1.your-objectstorage.com
+                  acl = private
+
+                  [storagebox]
+                  type = sftp
+                  host = ${STORAGEBOX_HOST}
+                  port = 23
+                  user = ${STORAGEBOX_USER}
+                  key_file = /etc/storagebox/ssh-key
+                  shell_type = none
+                  md5sum_command = none
+                  sha1sum_command = none
+
+                  [backup]
+                  type = compress
+                  remote = storagebox:ocis-backup
+                  CONF
+
+                  echo "Syncing S3 bucket to Storage Box (compressed)..."
+                  rclone sync s3:ocis-tr1ceracop backup: \
+                    --config /tmp/rclone/rclone.conf \
+                    --transfers 4 \
+                    -v
+
+                  rm -rf /tmp/rclone
+                  echo "Backup complete."
+              env:
+                - name: S3_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      key: accessKey
+                      name: ocis-s3-credentials
+                - name: S3_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      key: secretKey
+                      name: ocis-s3-credentials
+                - name: STORAGEBOX_HOST
+                  valueFrom:
+                    secretKeyRef:
+                      key: host
+                      name: ocis-storagebox-credentials
+                - name: STORAGEBOX_USER
+                  valueFrom:
+                    secretKeyRef:
+                      key: user
+                      name: ocis-storagebox-credentials
+              image: alpine:3.20
+              name: backup
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 128Mi
+              volumeMounts:
+                - mountPath: /etc/storagebox
+                  name: storagebox-ssh
+                  readOnly: true
+          restartPolicy: OnFailure
+          serviceAccountName: ocis-s3-backup
+          volumes:
+            - name: storagebox-ssh
+              secret:
+                defaultMode: 256
+                items:
+                  - key: ssh-key
+                    path: ssh-key
+                secretName: ocis-storagebox-credentials
+      ttlSecondsAfterFinished: 86400
+  schedule: 0 2 * * *
+  successfulJobsHistoryLimit: 3
diff --git a/rendered/envs/production/ocis/job-ocis-secret-init.yaml b/rendered/envs/production/ocis/job-ocis-secret-init.yaml
index a3ac4dc..afd035b 100644
--- a/rendered/envs/production/ocis/job-ocis-secret-init.yaml
+++ b/rendered/envs/production/ocis/job-ocis-secret-init.yaml
@@ -45,6 +45,11 @@ spec:
                 exit 1
               fi
 
+              if ! kubectl get secret ocis-storagebox-credentials -n "${NAMESPACE}" >/dev/null 2>&1; then
+                echo "ERROR: External secret ocis-storagebox-credentials must be created manually"
+                exit 1
+              fi
+
               # Admin user
               create_secret_if_missing ocis-admin-user \
                 --from-literal=password="$(gen_random 32)" \
diff --git a/rendered/envs/production/ocis/serviceaccount-ocis-s3-backup.yaml b/rendered/envs/production/ocis/serviceaccount-ocis-s3-backup.yaml
new file mode 100644
index 0000000..7026f8d
--- /dev/null
+++ b/rendered/envs/production/ocis/serviceaccount-ocis-s3-backup.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations:
+    a8r.io/repository: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
+  name: ocis-s3-backup
+  namespace: ocis