feat: Add automated backups for Forgejo (Postgres + git repos)

- CNPG Barman backup to Hetzner S3 (s3://k8s-and-chill-backups/forgejo/cnpg/)
- ScheduledBackup CR: daily at 2 AM, 30d retention, prefer-standby
- Git repo rclone sync to S3 (s3://k8s-and-chill-backups/forgejo/git/) via CronJob at 3 AM
- Requires secrets: forgejo-backup-s3 (S3 creds), hcloud-token (not used but created)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

envs/production/_apps/forgejo/argocd/replace-jobs.overlay.ytt.yaml

@@ -4,8 +4,12 @@
 ---
 #@overlay/match-child-defaults missing_ok=True
 spec:
+  ignoreDifferences:
+    - group: ""
+      kind: PersistentVolumeClaim
+      jsonPointers:
+        - /spec/volumeName
   syncPolicy:
-    #@overlay/match missing_ok=True
     syncOptions:
       #@overlay/append
-      - Replace=true
+      - RespectIgnoreDifferences=true

prototypes/forgejo/ytt/cnpg-cluster.ytt.yaml

@@ -27,6 +27,24 @@ spec:
     limits:
       memory: 512Mi
 
+  backup:
+    barmanObjectStore:
+      endpointURL: https://fsn1.your-objectstorage.com
+      destinationPath: s3://k8s-and-chill-backups/forgejo/cnpg
+      s3Credentials:
+        accessKeyId:
+          name: forgejo-backup-s3
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: forgejo-backup-s3
+          key: SECRET_ACCESS_KEY
+      wal:
+        compression: gzip
+      data:
+        compression: gzip
+    retentionPolicy: "30d"
+    target: prefer-standby
+
   postgresql:
     parameters:
       shared_buffers: "64MB"

prototypes/forgejo/ytt/cnpg-scheduled-backup.ytt.yaml

@@ -0,0 +1,17 @@
+#@ load("@ytt:data", "data")
+
+#@ ns = data.values.application.namespace
+
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: forgejo-daily-backup
+  namespace: #@ ns
+spec:
+  cluster:
+    name: forgejo-cnpg
+  schedule: "0 0 2 * * *"
+  method: barmanObjectStore
+  backupOwnerReference: cluster
+  target: prefer-standby

prototypes/forgejo/ytt/git-snapshot-cronjob.ytt.yaml

@@ -0,0 +1,87 @@
+#@ load("@ytt:data", "data")
+
+#@ ns = data.values.application.namespace
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: forgejo-git-backup
+  namespace: #@ ns
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: forgejo-git-backup
+  namespace: #@ ns
+spec:
+  schedule: "0 3 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 86400
+      template:
+        spec:
+          restartPolicy: OnFailure
+          serviceAccountName: forgejo-git-backup
+          nodeSelector:
+            kubernetes.io/hostname: ubuntu-4gb-nbg1-3
+          containers:
+            - name: backup
+              image: alpine:3.20
+              command:
+                - sh
+                - -c
+                - |
+                  set -e
+                  apk add --no-cache rclone > /dev/null 2>&1
+
+                  mkdir -p /tmp/rclone
+                  cat > /tmp/rclone/rclone.conf <<CONF
+                  [s3]
+                  type = s3
+                  provider = Other
+                  access_key_id = ${ACCESS_KEY_ID}
+                  secret_access_key = ${SECRET_ACCESS_KEY}
+                  endpoint = https://fsn1.your-objectstorage.com
+                  acl = private
+                  CONF
+
+                  echo "Syncing git repositories to S3..."
+                  rclone sync /data/git/ s3:k8s-and-chill-backups/forgejo/git/ \
+                    --config /tmp/rclone/rclone.conf \
+                    --transfers 4 \
+                    -v
+
+                  echo "Syncing gitea data (avatars, attachments, keys)..."
+                  rclone sync /data/gitea/ s3:k8s-and-chill-backups/forgejo/gitea/ \
+                    --config /tmp/rclone/rclone.conf \
+                    --exclude 'conf/**' \
+                    --exclude 'queues/**' \
+                    --transfers 4 \
+                    -v
+
+                  rm -rf /tmp/rclone
+                  echo "Backup complete."
+              env:
+                - name: ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-backup-s3
+                      key: ACCESS_KEY_ID
+                - name: SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: forgejo-backup-s3
+                      key: SECRET_ACCESS_KEY
+              volumeMounts:
+                - name: data
+                  mountPath: /data
+                  readOnly: true
+          volumes:
+            - name: data
+              persistentVolumeClaim:
+                claimName: forgejo-git-storage

rendered/argocd/production/app-forgejo.yaml

@@ -11,6 +11,11 @@ spec:
   destination:
     namespace: forgejo
     server: https://kubernetes.default.svc
+  ignoreDifferences:
+    - group: ""
+      jsonPointers:
+        - /spec/volumeName
+      kind: PersistentVolumeClaim
   project: env-production
   source:
     path: rendered/envs/production/forgejo
@@ -23,4 +28,4 @@ spec:
     syncOptions:
       - CreateNamespace=true
       - ServerSideApply=true
-      - Replace=true
+      - RespectIgnoreDifferences=true

rendered/envs/production/forgejo/cluster-forgejo-cnpg.yaml

@@ -6,6 +6,23 @@ metadata:
   name: forgejo-cnpg
   namespace: forgejo
 spec:
+  backup:
+    barmanObjectStore:
+      data:
+        compression: gzip
+      destinationPath: s3://k8s-and-chill-backups/forgejo/cnpg
+      endpointURL: https://fsn1.your-objectstorage.com
+      s3Credentials:
+        accessKeyId:
+          key: ACCESS_KEY_ID
+          name: forgejo-backup-s3
+        secretAccessKey:
+          key: SECRET_ACCESS_KEY
+          name: forgejo-backup-s3
+      wal:
+        compression: gzip
+    retentionPolicy: 30d
+    target: prefer-standby
   bootstrap:
     initdb:
       database: forgejo

rendered/envs/production/forgejo/cronjob-forgejo-git-backup.yaml

@@ -0,0 +1,77 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  annotations:
+    a8r.io/repository: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
+  name: forgejo-git-backup
+  namespace: forgejo
+spec:
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - command:
+                - sh
+                - -c
+                - |
+                  set -e
+                  apk add --no-cache rclone > /dev/null 2>&1
+
+                  mkdir -p /tmp/rclone
+                  cat > /tmp/rclone/rclone.conf <<CONF
+                  [s3]
+                  type = s3
+                  provider = Other
+                  access_key_id = ${ACCESS_KEY_ID}
+                  secret_access_key = ${SECRET_ACCESS_KEY}
+                  endpoint = https://fsn1.your-objectstorage.com
+                  acl = private
+                  CONF
+
+                  echo "Syncing git repositories to S3..."
+                  rclone sync /data/git/ s3:k8s-and-chill-backups/forgejo/git/ \
+                    --config /tmp/rclone/rclone.conf \
+                    --transfers 4 \
+                    -v
+
+                  echo "Syncing gitea data (avatars, attachments, keys)..."
+                  rclone sync /data/gitea/ s3:k8s-and-chill-backups/forgejo/gitea/ \
+                    --config /tmp/rclone/rclone.conf \
+                    --exclude 'conf/**' \
+                    --exclude 'queues/**' \
+                    --transfers 4 \
+                    -v
+
+                  rm -rf /tmp/rclone
+                  echo "Backup complete."
+              env:
+                - name: ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      key: ACCESS_KEY_ID
+                      name: forgejo-backup-s3
+                - name: SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      key: SECRET_ACCESS_KEY
+                      name: forgejo-backup-s3
+              image: alpine:3.20
+              name: backup
+              volumeMounts:
+                - mountPath: /data
+                  name: data
+                  readOnly: true
+          nodeSelector:
+            kubernetes.io/hostname: ubuntu-4gb-nbg1-3
+          restartPolicy: OnFailure
+          serviceAccountName: forgejo-git-backup
+          volumes:
+            - name: data
+              persistentVolumeClaim:
+                claimName: forgejo-git-storage
+      ttlSecondsAfterFinished: 86400
+  schedule: 0 3 * * *
+  successfulJobsHistoryLimit: 3

rendered/envs/production/forgejo/scheduledbackup-forgejo-daily-backup.yaml

@@ -0,0 +1,14 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  annotations:
+    a8r.io/repository: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
+  name: forgejo-daily-backup
+  namespace: forgejo
+spec:
+  backupOwnerReference: cluster
+  cluster:
+    name: forgejo-cnpg
+  method: barmanObjectStore
+  schedule: 0 0 2 * * *
+  target: prefer-standby

rendered/envs/production/forgejo/serviceaccount-forgejo-git-backup.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations:
+    a8r.io/repository: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
+  name: forgejo-git-backup
+  namespace: forgejo