refactor(prototypes): parameterize env-specific values for multi-env support

Extract domain, ingress class, TLS issuer, storage classes, S3 endpoints,
backup toggles, and forgejo node selector into env-data values. Each
prototype's app-data declares its subdomain alongside namespace; templates
compute host as <subdomain>.<cluster.domain>.

Schema is shape-only with safe defaults; production env-data sets values
explicitly. Backup CronJobs and external-secret prechecks gate on
backups.enabled and ocis.s3.external. Adds mkcert ClusterIssuer + precheck
Job for local-dev TLS, gated on cluster.tls.issuer == "mkcert".

forgejo argocd-deploy-key Job: REPO_URL/FORGEJO_URL moved to container env
vars to keep the script ytt-templatable; runtime behavior unchanged.

Production render verified byte-identical (excluding the deploy-key Job
env-var refactor and chart-volatile UUID ConfigMaps).
This commit is contained in:
Felix Wolf 2026-05-03 15:08:48 +02:00
parent e42ff64f7b
commit 279cd0d19f
24 changed files with 309 additions and 98 deletions

View file

@ -0,0 +1,37 @@
#@data/values-schema
---
#@overlay/match missing_ok=True
cluster:
domain: ""
ingress:
className: ""
tls:
issuer: ""
storageClass:
block: ""
local: ""
#@overlay/match missing_ok=True
backups:
enabled: false
s3:
endpoint: ""
region: ""
storagebox:
enabled: false
#@overlay/match missing_ok=True
ocis:
s3:
external: false
endpoint: ""
region: ""
bucket: ""
#@overlay/match missing_ok=True
forgejo:
sshPort: 22
#@schema/type any=True
nodeSelector: {}
backup:
s3Bucket: ""

View file

@ -1,3 +1,5 @@
#@ load("@ytt:overlay", "overlay")
#@data/values
---
environment:
@ -16,3 +18,36 @@ environment:
- proto: cloudnative-pg
- proto: metrics-server
- proto: ocis
cluster:
domain: tr1ceracop.de
ingress:
className: traefik
tls:
issuer: letsencrypt
storageClass:
block: hcloud-volumes
local: local-path
backups:
enabled: true
s3:
endpoint: https://fsn1.your-objectstorage.com
region: fsn1
storagebox:
enabled: true
ocis:
s3:
external: true
endpoint: https://nbg1.your-objectstorage.com
region: nbg1
bucket: ocis-tr1ceracop
forgejo:
sshPort: 222
#@overlay/replace
nodeSelector:
kubernetes.io/hostname: ubuntu-4gb-nbg1-3
backup:
s3Bucket: k8s-and-chill-backups

View file

@ -3,3 +3,4 @@
#@overlay/match-child-defaults missing_ok=True
application:
namespace: argocd
subdomain: argocd

View file

@ -1,10 +1,12 @@
#@ load("@ytt:data", "data")
#@ host = data.values.application.subdomain + "." + data.values.cluster.domain
---
crds:
install: true
keep: true
global:
domain: argocd.tr1ceracop.de
domain: #@ host
configs:
params:
@ -30,10 +32,10 @@ server:
enabled: true
ingress:
enabled: true
ingressClassName: traefik
ingressClassName: #@ data.values.cluster.ingress.className
tls: true
annotations:
cert-manager.io/cluster-issuer: letsencrypt
cert-manager.io/cluster-issuer: #@ data.values.cluster.tls.issuer
repoServer:
metrics:

View file

@ -1,5 +1,7 @@
#@ load("@ytt:overlay", "overlay")
#@ load("@ytt:data", "data")
#@ if data.values.cluster.tls.issuer == "letsencrypt":
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
@ -14,4 +16,5 @@ spec:
solvers:
- http01:
ingress:
ingressClassName: traefik
ingressClassName: #@ data.values.cluster.ingress.className
#@ end

View file

@ -0,0 +1,85 @@
#@ load("@ytt:data", "data")
#@ ns = data.values.application.namespace
#@ if data.values.cluster.tls.issuer == "mkcert":
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: mkcert-ca-precheck
namespace: #@ ns
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: mkcert-ca-precheck
namespace: #@ ns
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: mkcert-ca-precheck
namespace: #@ ns
subjects:
- kind: ServiceAccount
name: mkcert-ca-precheck
namespace: #@ ns
roleRef:
kind: Role
name: mkcert-ca-precheck
apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
name: mkcert-ca-precheck
namespace: #@ ns
annotations:
argocd.argoproj.io/sync-wave: "-1"
argocd.argoproj.io/sync-options: Replace=true
spec:
ttlSecondsAfterFinished: 300
template:
spec:
serviceAccountName: mkcert-ca-precheck
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 65532
runAsGroup: 65532
seccompProfile:
type: RuntimeDefault
containers:
- name: precheck
image: alpine/k8s:1.32.3
command:
- sh
- -c
- |
set -e
if ! kubectl get secret mkcert-ca -n "${NAMESPACE}" >/dev/null 2>&1; then
echo "ERROR: External secret mkcert-ca must be created in ${NAMESPACE} before deploying cert-manager."
echo "Run: mkcert -install && kubectl -n ${NAMESPACE} create secret tls mkcert-ca --cert=\"\$(mkcert -CAROOT)/rootCA.pem\" --key=\"\$(mkcert -CAROOT)/rootCA-key.pem\""
exit 1
fi
echo "OK: mkcert-ca exists"
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
#@ end

View file

@ -0,0 +1,12 @@
#@ load("@ytt:data", "data")
#@ if data.values.cluster.tls.issuer == "mkcert":
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: mkcert
spec:
ca:
secretName: mkcert-ca
#@ end

View file

@ -3,3 +3,4 @@
#@overlay/match-child-defaults missing_ok=True
application:
namespace: forgejo
subdomain: git

View file

@ -1,3 +1,5 @@
#@ load("@ytt:data", "data")
#@ host = data.values.application.subdomain + "." + data.values.cluster.domain
---
strategy:
type: Recreate
@ -17,16 +19,16 @@ persistence:
ingress:
enabled: true
hosts:
- host: git.tr1ceracop.de
- host: #@ host
paths:
- path: /
pathType: Prefix
tls:
- secretName: forgejo-tls
hosts:
- git.tr1ceracop.de
- #@ host
annotations:
cert-manager.io/cluster-issuer: letsencrypt
cert-manager.io/cluster-issuer: #@ data.values.cluster.tls.issuer
service:
ssh:
@ -58,9 +60,9 @@ gitea:
queue:
TYPE: level
server:
DOMAIN: git.tr1ceracop.de
ROOT_URL: https://git.tr1ceracop.de/
SSH_PORT: 222
DOMAIN: #@ host
ROOT_URL: #@ "https://{}/".format(host)
SSH_PORT: #@ data.values.forgejo.sshPort
service:
DISABLE_REGISTRATION: true
actions:

View file

@ -1,6 +1,9 @@
#@ load("@ytt:data", "data")
#@ ns = data.values.application.namespace
#@ host = data.values.application.subdomain + "." + data.values.cluster.domain
#@ repo_url = "ssh://git@" + host + ":" + str(data.values.forgejo.sshPort) + "/gitea_admin/k8s-and-chill.git"
#@ forgejo_url = "https://" + host
---
apiVersion: v1
@ -51,6 +54,15 @@ spec:
containers:
- name: init
image: alpine/k8s:1.32.3
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: REPO_URL
value: #@ repo_url
- name: FORGEJO_URL
value: #@ forgejo_url
command:
- sh
- -c
@ -59,8 +71,6 @@ spec:
ARGOCD_NS="argocd"
REPO_SECRET="forgejo-repo"
REPO_URL="ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git"
FORGEJO_URL="https://git.tr1ceracop.de"
REPO_OWNER="gitea_admin"
REPO_NAME="k8s-and-chill"
@ -142,8 +152,3 @@ spec:
EOSECRET
echo "Created ArgoCD repository secret ${REPO_SECRET} in ${ARGOCD_NS}"
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace

View file

@ -18,7 +18,7 @@ spec:
storage:
size: 5Gi
storageClass: hcloud-volumes
storageClass: #@ data.values.cluster.storageClass.block
resources:
requests:
@ -27,10 +27,11 @@ spec:
limits:
memory: 512Mi
#@ if data.values.backups.enabled:
backup:
barmanObjectStore:
endpointURL: https://fsn1.your-objectstorage.com
destinationPath: s3://k8s-and-chill-backups/forgejo/cnpg
endpointURL: #@ data.values.backups.s3.endpoint
destinationPath: #@ "s3://{}/forgejo/cnpg".format(data.values.forgejo.backup.s3Bucket)
s3Credentials:
accessKeyId:
name: forgejo-backup-s3
@ -44,6 +45,7 @@ spec:
compression: gzip
retentionPolicy: "30d"
target: prefer-standby
#@ end
postgresql:
parameters:

View file

@ -2,6 +2,7 @@
#@ ns = data.values.application.namespace
#@ if data.values.backups.enabled:
---
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
@ -15,3 +16,4 @@ spec:
method: barmanObjectStore
backupOwnerReference: cluster
target: prefer-standby
#@ end

View file

@ -13,7 +13,7 @@ metadata:
spec:
accessModes:
- ReadWriteOnce
storageClassName: hcloud-volumes
storageClassName: #@ data.values.cluster.storageClass.block
resources:
requests:
storage: 20Gi

View file

@ -2,6 +2,43 @@
#@ ns = data.values.application.namespace
#@ s3_endpoint = data.values.backups.s3.endpoint
#@ s3_bucket = data.values.forgejo.backup.s3Bucket
#@ backup_script = """\
#@ set -e
#@ apk add --no-cache rclone > /dev/null 2>&1
#@
#@ mkdir -p /tmp/rclone
#@ cat > /tmp/rclone/rclone.conf <<CONF
#@ [s3]
#@ type = s3
#@ provider = Other
#@ access_key_id = ${ACCESS_KEY_ID}
#@ secret_access_key = ${SECRET_ACCESS_KEY}
#@ endpoint = """ + s3_endpoint + """
#@ acl = private
#@ CONF
#@
#@ echo "Syncing git repositories to S3..."
#@ rclone sync /data/git/ s3:""" + s3_bucket + """/forgejo/git/ \\
#@ --config /tmp/rclone/rclone.conf \\
#@ --transfers 4 \\
#@ -v
#@
#@ echo "Syncing gitea data (avatars, attachments, keys)..."
#@ rclone sync /data/gitea/ s3:""" + s3_bucket + """/forgejo/gitea/ \\
#@ --config /tmp/rclone/rclone.conf \\
#@ --exclude 'conf/**' \\
#@ --exclude 'queues/**' \\
#@ --transfers 4 \\
#@ -v
#@
#@ rm -rf /tmp/rclone
#@ echo "Backup complete."
#@ """
#@ if data.values.backups.enabled:
---
apiVersion: v1
kind: ServiceAccount
@ -27,45 +64,14 @@ spec:
spec:
restartPolicy: OnFailure
serviceAccountName: forgejo-git-backup
nodeSelector:
kubernetes.io/hostname: ubuntu-4gb-nbg1-3
nodeSelector: #@ data.values.forgejo.nodeSelector
containers:
- name: backup
image: alpine:3.20
command:
- sh
- -c
- |
set -e
apk add --no-cache rclone > /dev/null 2>&1
mkdir -p /tmp/rclone
cat > /tmp/rclone/rclone.conf <<CONF
[s3]
type = s3
provider = Other
access_key_id = ${ACCESS_KEY_ID}
secret_access_key = ${SECRET_ACCESS_KEY}
endpoint = https://fsn1.your-objectstorage.com
acl = private
CONF
echo "Syncing git repositories to S3..."
rclone sync /data/git/ s3:k8s-and-chill-backups/forgejo/git/ \
--config /tmp/rclone/rclone.conf \
--transfers 4 \
-v
echo "Syncing gitea data (avatars, attachments, keys)..."
rclone sync /data/gitea/ s3:k8s-and-chill-backups/forgejo/gitea/ \
--config /tmp/rclone/rclone.conf \
--exclude 'conf/**' \
--exclude 'queues/**' \
--transfers 4 \
-v
rm -rf /tmp/rclone
echo "Backup complete."
- #@ backup_script
env:
- name: ACCESS_KEY_ID
valueFrom:
@ -85,3 +91,4 @@ spec:
- name: data
persistentVolumeClaim:
claimName: forgejo-git-storage
#@ end

View file

@ -1,4 +1,7 @@
#@ load("@ytt:overlay", "overlay")
#@ load("@ytt:data", "data")
#@ host = data.values.application.subdomain + "." + data.values.cluster.domain
#@overlay/match by=overlay.subset({"kind": "Ingress"}), expects="0+"
---
@ -6,7 +9,7 @@ apiVersion: networking.k8s.io/v1
spec:
#@overlay/replace
rules:
- host: git.tr1ceracop.de
- host: #@ host
http:
paths:
- path: /

View file

@ -1,4 +1,5 @@
#@ load("@ytt:overlay", "overlay")
#@ load("@ytt:data", "data")
#! Add hostPort 22 to the SSH container port and pin to the DNS target node
#@overlay/match by=overlay.subset({"kind": "Deployment", "metadata": {"name": "forgejo"}})
@ -7,11 +8,10 @@ spec:
template:
spec:
#@overlay/match missing_ok=True
nodeSelector:
kubernetes.io/hostname: ubuntu-4gb-nbg1-3
nodeSelector: #@ data.values.forgejo.nodeSelector
containers:
#@overlay/match by=overlay.subset({"name": "forgejo"})
- ports:
#@overlay/match by=overlay.subset({"name": "ssh"})
#@overlay/match-child-defaults missing_ok=True
- hostPort: 222
- hostPort: #@ data.values.forgejo.sshPort

View file

@ -3,3 +3,4 @@
#@overlay/match-child-defaults missing_ok=True
application:
namespace: monitoring
subdomain: grafana

View file

@ -1,3 +1,5 @@
#@ load("@ytt:data", "data")
#@ host = data.values.application.subdomain + "." + data.values.cluster.domain
---
admin:
existingSecret: grafana-admin
@ -15,19 +17,19 @@ resources:
persistence:
enabled: true
size: 2Gi
storageClassName: local-path
storageClassName: #@ data.values.cluster.storageClass.local
ingress:
enabled: true
ingressClassName: traefik
ingressClassName: #@ data.values.cluster.ingress.className
hosts:
- grafana.tr1ceracop.de
- #@ host
tls:
- secretName: grafana-tls
hosts:
- grafana.tr1ceracop.de
- #@ host
annotations:
cert-manager.io/cluster-issuer: letsencrypt
cert-manager.io/cluster-issuer: #@ data.values.cluster.tls.issuer
datasources:
datasources.yaml:

View file

@ -3,3 +3,4 @@
#@overlay/match-child-defaults missing_ok=True
application:
namespace: ocis
subdomain: drive

View file

@ -1,15 +1,17 @@
#@ load("@ytt:data", "data")
#@ host = data.values.application.subdomain + "." + data.values.cluster.domain
---
externalDomain: drive.tr1ceracop.de
externalDomain: #@ host
ingress:
enabled: true
ingressClassName: traefik
ingressClassName: #@ data.values.cluster.ingress.className
annotations:
cert-manager.io/cluster-issuer: letsencrypt
cert-manager.io/cluster-issuer: #@ data.values.cluster.tls.issuer
tls:
- secretName: ocis-tls
hosts:
- drive.tr1ceracop.de
- #@ host
features:
emailNotifications:
@ -51,12 +53,12 @@ services:
driver: s3ng
driverConfig:
s3ng:
endpoint: https://nbg1.your-objectstorage.com
region: nbg1
bucket: ocis-tr1ceracop
endpoint: #@ data.values.ocis.s3.endpoint
region: #@ data.values.ocis.s3.region
bucket: #@ data.values.ocis.s3.bucket
persistence:
enabled: true
storageClassName: hcloud-volumes
storageClassName: #@ data.values.cluster.storageClass.block
size: 10Gi
accessModes:
- ReadWriteOnce
@ -64,7 +66,7 @@ services:
storagesystem:
persistence:
enabled: true
storageClassName: hcloud-volumes
storageClassName: #@ data.values.cluster.storageClass.block
size: 10Gi
accessModes:
- ReadWriteOnce
@ -76,7 +78,7 @@ services:
cpu: 10m
persistence:
enabled: true
storageClassName: hcloud-volumes
storageClassName: #@ data.values.cluster.storageClass.block
size: 10Gi
accessModes:
- ReadWriteOnce
@ -88,7 +90,7 @@ services:
cpu: 10m
persistence:
enabled: true
storageClassName: local-path
storageClassName: #@ data.values.cluster.storageClass.local
size: 1Gi
accessModes:
- ReadWriteOnce
@ -96,25 +98,25 @@ services:
search:
persistence:
enabled: true
storageClassName: local-path
storageClassName: #@ data.values.cluster.storageClass.local
size: 5Gi
accessModes:
- ReadWriteOnce
web:
# GOTCHA: if this PVC is recreated, /branding/logo POST/DELETE will
# 500 with "permission denied". The chart mounts an `apps` emptyDir
# at /var/lib/ocis/web/assets/apps; kubelet auto-creates the parent
# dirs as root:root 0755 *after* fsGroup runs, and local-path's
# hostPath PV doesn't get fsGroup recursion — so user 1000 can't
# mkdir themes/ to store the uploaded logo. Was masked while ocis
# ran as PSS=privileged (root); surfaced after PSS=restricted.
# Remediation: one-shot privileged Job in kube-system, hostPath-
# mount the local-path PV directory, `chown -R 1000:1000` it.
# Permanent fix: switch to a CSI storageClass (hcloud-volumes).
#! GOTCHA: if this PVC is recreated, /branding/logo POST/DELETE will
#! 500 with "permission denied". The chart mounts an `apps` emptyDir
#! at /var/lib/ocis/web/assets/apps; kubelet auto-creates the parent
#! dirs as root:root 0755 *after* fsGroup runs, and local-path's
#! hostPath PV doesn't get fsGroup recursion — so user 1000 can't
#! mkdir themes/ to store the uploaded logo. Was masked while ocis
#! ran as PSS=privileged (root); surfaced after PSS=restricted.
#! Remediation: one-shot privileged Job in kube-system, hostPath-
#! mount the local-path PV directory, `chown -R 1000:1000` it.
#! Permanent fix: switch to a CSI storageClass (hcloud-volumes).
persistence:
enabled: true
storageClassName: local-path
storageClassName: #@ data.values.cluster.storageClass.local
size: 1Gi
accessModes:
- ReadWriteOnce
@ -131,7 +133,7 @@ services:
cpu: 10m
persistence:
enabled: true
storageClassName: local-path
storageClassName: #@ data.values.cluster.storageClass.local
size: 2Gi
accessModes:
- ReadWriteOnce

View file

@ -2,6 +2,15 @@
#@ ns = data.values.application.namespace
#@ secrets = []
#@ if data.values.ocis.s3.external:
#@ secrets.append("ocis-s3-credentials")
#@ end
#@ if data.values.backups.enabled and data.values.backups.storagebox.enabled:
#@ secrets.append("ocis-storagebox-credentials")
#@ end
#@ if len(secrets) > 0:
---
apiVersion: v1
kind: ServiceAccount
@ -72,15 +81,7 @@ spec:
command:
- sh
- -c
- |
set -e
for s in ocis-s3-credentials ocis-storagebox-credentials; do
if ! kubectl get secret "$s" -n "${NAMESPACE}" >/dev/null 2>&1; then
echo "ERROR: External secret $s must be created manually before deploying ocis"
exit 1
fi
echo "OK: $s exists"
done
- #@ "set -e\nfor s in " + " ".join(secrets) + "; do\n if ! kubectl get secret \"$s\" -n \"${NAMESPACE}\" >/dev/null 2>&1; then\n echo \"ERROR: External secret $s must be created manually before deploying ocis\"\n exit 1\n fi\n echo \"OK: $s exists\"\ndone\n"
env:
- name: NAMESPACE
valueFrom:
@ -92,3 +93,4 @@ spec:
capabilities:
drop:
- ALL
#@ end

View file

@ -2,6 +2,7 @@
#@ ns = data.values.application.namespace
#@ if data.values.backups.enabled and data.values.backups.storagebox.enabled:
---
apiVersion: v1
kind: ServiceAccount
@ -110,3 +111,4 @@ spec:
- key: ssh-key
path: ssh-key
defaultMode: 0440
#@ end

View file

@ -19,6 +19,7 @@
#@ {"app": "storagesystem", "pvc": "storagesystem-data"},
#@ ]
#@ if data.values.backups.enabled and data.values.backups.storagebox.enabled:
#@ for t in targets:
---
apiVersion: batch/v1
@ -118,3 +119,4 @@ spec:
path: ssh-key
defaultMode: 0440
#@ end
#@ end

View file

@ -19,8 +19,6 @@ spec:
ARGOCD_NS="argocd"
REPO_SECRET="forgejo-repo"
REPO_URL="ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git"
FORGEJO_URL="https://git.tr1ceracop.de"
REPO_OWNER="gitea_admin"
REPO_NAME="k8s-and-chill"
@ -107,6 +105,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: REPO_URL
value: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
- name: FORGEJO_URL
value: https://git.tr1ceracop.de
image: alpine/k8s:1.32.3
name: init
restartPolicy: OnFailure