Adds a self-contained minikube environment for local development and
testing alongside the existing production env.
env: minikube
- cluster.domain: minikube (browser DNS routes *.minikube → minikube ip)
- tls issuer: mkcert (CA-signed via cert-manager mkcert ClusterIssuer)
- storageClass: standard (minikube hostpath provisioner)
- backups disabled; storagebox disabled
- excludes argocd, forgejo, hcloud-csi (manual kubectl apply for testing)
prototypes/garage:
- hand-rolled S3-compatible object store (single Deployment + PVC)
- mittwald-generated rpc_secret + admin_token (hex)
- PostSync init Job: assigns cluster layout, ensures bucket and access
key, writes ocis-s3-credentials cross-namespace into ocis ns
- idempotent: skips if k8s secret already populated; otherwise rotates
the garage key (admin API only returns secretAccessKey on create)
- cross-ns RBAC re-pinned via zz-cross-ns-rbac-fix overlay (ns.ytt.yaml
clobbers explicit namespace fields)
ocis:
- new admin-user-id init Job ensures secret.user-id is a valid UUID v4
(mittwald can't generate UUIDs; ocis-settings rejects non-UUID ids)
- mittwald no longer manages user-id; existing prod UUIDs preserved
- insecure flag (oidcIdpInsecure / ocisHttpApiInsecure / ocmInsecure)
parameterized; defaults to false; minikube sets true for self-signed
OIDC issuer URL trust
other prototypes:
- victoria-metrics-single helm values ytt-ified (storageClassName)
- grafana admin secret now generated by mittwald (was hand-created in
prod; manifest is no-op there since mittwald only fills empty fields)
flake.nix: minikube + docker + postgresql added to dev shell.
129 lines
5.7 KiB
YAML
129 lines
5.7 KiB
YAML
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
annotations:
|
|
a8r.io/repository: ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git
|
|
argocd.argoproj.io/sync-options: Replace=true,Force=true
|
|
name: garage-init
|
|
namespace: garage
|
|
spec:
|
|
backoffLimit: 30
|
|
template:
|
|
spec:
|
|
containers:
|
|
- args:
|
|
- |
|
|
set -eu
|
|
|
|
ADMIN_TOKEN=$(cat /etc/garage/admin_token)
|
|
AUTH="Authorization: Bearer ${ADMIN_TOKEN}"
|
|
ADMIN="http://garage.garage.svc:3903"
|
|
OCIS_NS="ocis"
|
|
OCIS_SECRET="ocis-s3-credentials"
|
|
BUCKET_NAME="ocis-minikube"
|
|
KEY_NAME="ocis"
|
|
|
|
echo "[garage-init] checking k8s secret ${OCIS_SECRET} in ${OCIS_NS}..."
|
|
EXISTING_AK=$(kubectl get secret "${OCIS_SECRET}" -n "${OCIS_NS}" -o jsonpath='{.data.accessKey}' 2>/dev/null || echo "")
|
|
EXISTING_SK=$(kubectl get secret "${OCIS_SECRET}" -n "${OCIS_NS}" -o jsonpath='{.data.secretKey}' 2>/dev/null || echo "")
|
|
if [ -n "${EXISTING_AK}" ] && [ -n "${EXISTING_SK}" ]; then
|
|
echo "[garage-init] ${OCIS_SECRET} already populated; skipping (idempotent exit)"
|
|
exit 0
|
|
fi
|
|
|
|
echo "[garage-init] waiting for admin API..."
|
|
until curl -fsS "${ADMIN}/health" >/dev/null 2>&1; do sleep 2; done
|
|
|
|
echo "[garage-init] checking cluster layout..."
|
|
STATUS=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/status")
|
|
NODE_ID=$(echo "${STATUS}" | jq -r '.nodes[0].id')
|
|
CUR_VERSION=$(echo "${STATUS}" | jq -r '.layoutVersion // 0')
|
|
|
|
if [ "${CUR_VERSION}" = "0" ] || [ "${CUR_VERSION}" = "null" ]; then
|
|
echo "[garage-init] applying initial layout for node ${NODE_ID}"
|
|
curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
|
|
-d "[{\"id\":\"${NODE_ID}\",\"zone\":\"dc1\",\"capacity\":1073741824,\"tags\":[]}]" \
|
|
"${ADMIN}/v1/layout"
|
|
curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
|
|
-d '{"version":1}' \
|
|
"${ADMIN}/v1/layout/apply"
|
|
echo "[garage-init] layout applied, waiting for cluster ready..."
|
|
for i in $(seq 1 30); do
|
|
READY=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/status" | jq -r '.layoutVersion')
|
|
if [ "${READY}" = "1" ]; then break; fi
|
|
sleep 2
|
|
done
|
|
else
|
|
echo "[garage-init] layout already at version ${CUR_VERSION}, skipping"
|
|
fi
|
|
|
|
echo "[garage-init] ensuring bucket ${BUCKET_NAME}..."
|
|
BUCKET_ID=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/bucket?globalAlias=${BUCKET_NAME}" 2>/dev/null | jq -r '.id // empty')
|
|
if [ -z "${BUCKET_ID}" ]; then
|
|
BUCKET_ID=$(curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
|
|
-d "{\"globalAlias\":\"${BUCKET_NAME}\"}" \
|
|
"${ADMIN}/v1/bucket" | jq -r '.id')
|
|
echo "[garage-init] created bucket ${BUCKET_ID}"
|
|
else
|
|
echo "[garage-init] bucket exists: ${BUCKET_ID}"
|
|
fi
|
|
|
|
echo "[garage-init] resetting key ${KEY_NAME} (need fresh secret)..."
|
|
EXISTING_KEY_ID=$(curl -fsS -H "${AUTH}" "${ADMIN}/v1/key?search=${KEY_NAME}" 2>/dev/null | jq -r '.accessKeyId // empty')
|
|
if [ -n "${EXISTING_KEY_ID}" ]; then
|
|
echo "[garage-init] deleting stale key ${EXISTING_KEY_ID}"
|
|
curl -fsS -X DELETE -H "${AUTH}" "${ADMIN}/v1/key?id=${EXISTING_KEY_ID}" >/dev/null
|
|
fi
|
|
KEY_INFO=$(curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
|
|
-d "{\"name\":\"${KEY_NAME}\"}" \
|
|
"${ADMIN}/v1/key")
|
|
ACCESS_KEY=$(echo "${KEY_INFO}" | jq -r '.accessKeyId')
|
|
SECRET_KEY=$(echo "${KEY_INFO}" | jq -r '.secretAccessKey')
|
|
if [ -z "${ACCESS_KEY}" ] || [ -z "${SECRET_KEY}" ] || [ "${SECRET_KEY}" = "null" ]; then
|
|
echo "[garage-init] failed to create key: ${KEY_INFO}"
|
|
exit 1
|
|
fi
|
|
echo "[garage-init] created key ${ACCESS_KEY}"
|
|
|
|
echo "[garage-init] granting bucket permissions..."
|
|
curl -fsS -X POST -H "${AUTH}" -H 'Content-Type: application/json' \
|
|
-d "{\"bucketId\":\"${BUCKET_ID}\",\"accessKeyId\":\"${ACCESS_KEY}\",\"permissions\":{\"read\":true,\"write\":true,\"owner\":true}}" \
|
|
"${ADMIN}/v1/bucket/allow" >/dev/null
|
|
|
|
echo "[garage-init] writing ${OCIS_SECRET} to ${OCIS_NS}..."
|
|
kubectl create secret generic "${OCIS_SECRET}" \
|
|
-n "${OCIS_NS}" \
|
|
--from-literal=accessKey="${ACCESS_KEY}" \
|
|
--from-literal=secretKey="${SECRET_KEY}" \
|
|
--dry-run=client -o yaml | kubectl apply -f -
|
|
|
|
echo "[garage-init] done."
|
|
command:
|
|
- sh
|
|
- -c
|
|
image: alpine/k8s:1.32.3
|
|
name: init
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
readOnlyRootFilesystem: true
|
|
volumeMounts:
|
|
- mountPath: /etc/garage
|
|
name: garage-secrets
|
|
readOnly: true
|
|
restartPolicy: OnFailure
|
|
securityContext:
|
|
runAsGroup: 65532
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
serviceAccountName: garage-init
|
|
volumes:
|
|
- name: garage-secrets
|
|
secret:
|
|
secretName: garage-secrets
|
|
ttlSecondsAfterFinished: 300
|