k8s-and-chill/rendered/envs/minikube/garage/job-garage-init.yaml
Felix Wolf fe51c8c1bc feat(minikube): add minikube environment with garage S3 backend
Adds a self-contained minikube environment for local development and
testing alongside the existing production env.

env: minikube
  - cluster.domain: minikube (browser DNS routes *.minikube → minikube ip)
  - tls issuer: mkcert (CA-signed via cert-manager mkcert ClusterIssuer)
  - storageClass: standard (minikube hostpath provisioner)
  - backups disabled; storagebox disabled
  - excludes argocd, forgejo, hcloud-csi (manual kubectl apply for testing)

prototypes/garage:
  - hand-rolled S3-compatible object store (single Deployment + PVC)
  - mittwald-generated rpc_secret + admin_token (hex)
  - PostSync init Job: assigns cluster layout, ensures bucket and access
    key, writes ocis-s3-credentials cross-namespace into ocis ns
  - idempotent: skips if k8s secret already populated; otherwise rotates
    the garage key (admin API only returns secretAccessKey on create)
  - cross-ns RBAC re-pinned via zz-cross-ns-rbac-fix overlay (ns.ytt.yaml
    clobbers explicit namespace fields)

ocis:
  - new admin-user-id init Job ensures secret.user-id is a valid UUID v4
    (mittwald can't generate UUIDs; ocis-settings rejects non-UUID ids)
  - mittwald no longer manages user-id; existing prod UUIDs preserved
  - insecure flag (oidcIdpInsecure / ocisHttpApiInsecure / ocmInsecure)
    parameterized; defaults to false; minikube sets true for self-signed
    OIDC issuer URL trust

other prototypes:
  - victoria-metrics-single helm values ytt-ified (storageClassName)
  - grafana admin secret now generated by mittwald (was hand-created in
    prod; manifest is no-op there since mittwald only fills empty fields)

flake.nix: minikube + docker + postgresql added to dev shell.
2026-05-03 17:23:57 +02:00

129 lines
5.7 KiB
YAML

apiVersion: batch/v1
kind: Job
metadata:
annotations:
a8r.io/repository: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
argocd.argoproj.io/sync-options: Replace=true,Force=true
name: garage-init
namespace: garage
spec:
backoffLimit: 30
template:
spec:
containers:
- args:
- |
set -eu
ADMIN_TOKEN=$(cat /etc/garage/admin_token)
AUTH="Authorization: Bearer ${ADMIN_TOKEN}"
ADMIN="http://garage.garage.svc:3903"
OCIS_NS="ocis"
OCIS_SECRET="ocis-s3-credentials"
BUCKET_NAME="ocis-minikube"
KEY_NAME="ocis"
echo "[garage-init] checking k8s secret ${OCIS_SECRET} in ${OCIS_NS}..."
EXISTING_AK=$(kubectl get secret "${OCIS_SECRET}" -n "${OCIS_NS}" -o jsonpath='{.data.accessKey}' 2>/dev/null || echo "")
EXISTING_SK=$(kubectl get secret "${OCIS_SECRET}" -n "${OCIS_NS}" -o jsonpath='{.data.secretKey}' 2>/dev/null || echo "")
if [ -n "${EXISTING_AK}" ] && [ -n "${EXISTING_SK}" ]; then
echo "[garage-init] ${OCIS_SECRET} already populated; skipping (idempotent exit)"
exit 0
fi
echo "[garage-init] waiting for admin API..."
until curl -fsS "${ADMIN}/health" >/dev/null 2>&1; do sleep 2; done
echo "[garage-init] checking cluster layout..."
STATUS=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/status")
NODE_ID=$(echo "${STATUS}" | jq -r '.nodes[0].id')
CUR_VERSION=$(echo "${STATUS}" | jq -r '.layoutVersion // 0')
if [ "${CUR_VERSION}" = "0" ] || [ "${CUR_VERSION}" = "null" ]; then
echo "[garage-init] applying initial layout for node ${NODE_ID}"
curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
-d "[{\"id\":\"${NODE_ID}\",\"zone\":\"dc1\",\"capacity\":1073741824,\"tags\":[]}]" \
"${ADMIN}/v1/layout"
curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
-d '{"version":1}' \
"${ADMIN}/v1/layout/apply"
echo "[garage-init] layout applied, waiting for cluster ready..."
for i in $(seq 1 30); do
READY=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/status" | jq -r '.layoutVersion')
if [ "${READY}" = "1" ]; then break; fi
sleep 2
done
else
echo "[garage-init] layout already at version ${CUR_VERSION}, skipping"
fi
echo "[garage-init] ensuring bucket ${BUCKET_NAME}..."
BUCKET_ID=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/bucket?globalAlias=${BUCKET_NAME}" 2>/dev/null | jq -r '.id // empty')
if [ -z "${BUCKET_ID}" ]; then
BUCKET_ID=$(curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
-d "{\"globalAlias\":\"${BUCKET_NAME}\"}" \
"${ADMIN}/v1/bucket" | jq -r '.id')
echo "[garage-init] created bucket ${BUCKET_ID}"
else
echo "[garage-init] bucket exists: ${BUCKET_ID}"
fi
echo "[garage-init] resetting key ${KEY_NAME} (need fresh secret)..."
EXISTING_KEY_ID=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/key?search=${KEY_NAME}" 2>/dev/null | jq -r '.accessKeyId // empty')
if [ -n "${EXISTING_KEY_ID}" ]; then
echo "[garage-init] deleting stale key ${EXISTING_KEY_ID}"
curl -fsS -X DELETE -H "${AUTH}" "${ADMIN}/v1/key?id=${EXISTING_KEY_ID}" >/dev/null
fi
KEY_INFO=$(curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
-d "{\"name\":\"${KEY_NAME}\"}" \
"${ADMIN}/v1/key")
ACCESS_KEY=$(echo "${KEY_INFO}" | jq -r '.accessKeyId')
SECRET_KEY=$(echo "${KEY_INFO}" | jq -r '.secretAccessKey')
if [ -z "${ACCESS_KEY}" ] || [ -z "${SECRET_KEY}" ] || [ "${SECRET_KEY}" = "null" ]; then
echo "[garage-init] failed to create key: ${KEY_INFO}"
exit 1
fi
echo "[garage-init] created key ${ACCESS_KEY}"
echo "[garage-init] granting bucket permissions..."
curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
-d "{\"bucketId\":\"${BUCKET_ID}\",\"accessKeyId\":\"${ACCESS_KEY}\",\"permissions\":{\"read\":true,\"write\":true,\"owner\":true}}" \
"${ADMIN}/v1/bucket/allow" >/dev/null
echo "[garage-init] writing ${OCIS_SECRET} to ${OCIS_NS}..."
kubectl create secret generic "${OCIS_SECRET}" \
-n "${OCIS_NS}" \
--from-literal=accessKey="${ACCESS_KEY}" \
--from-literal=secretKey="${SECRET_KEY}" \
--dry-run=client -o yaml | kubectl apply -f -
echo "[garage-init] done."
command:
- sh
- -c
image: alpine/k8s:1.32.3
name: init
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /etc/garage
name: garage-secrets
readOnly: true
restartPolicy: OnFailure
securityContext:
runAsGroup: 65532
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
serviceAccountName: garage-init
volumes:
- name: garage-secrets
secret:
secretName: garage-secrets
ttlSecondsAfterFinished: 300