fix: Make deploy key job fully idempotent with SSH known hosts

Restructure the argocd-deploy-key-init job so each step (known hosts,
deploy key registration, secret creation) is independently idempotent.
Add ssh-keyscan of Forgejo host key and patch ArgoCD known hosts
ConfigMap. Use kubectl apply with inline YAML to create the repo secret
with the argocd label in a single atomic step. Switch images from
bitnami/kubectl to alpine/k8s.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

prototypes/forgejo/ytt/argocd-deploy-key-job.ytt.yaml

@@ -18,6 +18,9 @@ rules:
   - apiGroups: [""]
     resources: ["secrets"]
     verbs: ["get", "create"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "patch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -54,37 +57,23 @@ spec:
             - |
               set -e
 
-              apk add --no-cache openssh-keygen > /dev/null 2>&1
+              apk add --no-cache openssh-keygen openssh-client > /dev/null 2>&1
 
               ARGOCD_NS="argocd"
               REPO_SECRET="forgejo-repo"
               REPO_URL="ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git"
-              FORGEJO_URL="https://git.tr1ceracop.de"
+              FORGEJO_HOST="git.tr1ceracop.de"
+              FORGEJO_SSH_PORT="222"
+              FORGEJO_URL="https://${FORGEJO_HOST}"
               REPO_OWNER="gitea_admin"
               REPO_NAME="k8s-and-chill"
-
+              KNOWN_HOSTS_CM="argocd-ssh-known-hosts-cm"
-              # Check if ArgoCD repo secret already exists
-              if kubectl get secret "${REPO_SECRET}" -n "${ARGOCD_NS}" >/dev/null 2>&1; then
-                echo "Secret ${REPO_SECRET} already exists in ${ARGOCD_NS}, skipping"
-                exit 0
-              fi
-
-              # Read admin credentials from forgejo-admin-secret
-              ADMIN_USER=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.username}' | base64 -d)
-              ADMIN_PASS=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.password}' | base64 -d)
-
-              # Generate ed25519 SSH keypair
-              KEYDIR=$(mktemp -d)
-              ssh-keygen -t ed25519 -f "${KEYDIR}/id_ed25519" -N "" -q
-              PRIVKEY=$(cat "${KEYDIR}/id_ed25519")
-              PUBKEY=$(cat "${KEYDIR}/id_ed25519.pub")
-              rm -rf "${KEYDIR}"
 
               # Wait for Forgejo to be ready
               echo "Waiting for Forgejo to be ready..."
               for i in $(seq 1 60); do
                 if curl -sk "${FORGEJO_URL}/api/v1/version" >/dev/null 2>&1; then
-                  echo "Forgejo is ready"
+                  echo "Forgejo HTTPS is ready"
                   break
                 fi
                 if [ "$i" -eq 60 ]; then
@@ -94,37 +83,79 @@ spec:
                 sleep 5
               done
 
-              # Register deploy key via Forgejo API
+              # Step 1: Add Forgejo SSH host key to ArgoCD known hosts
-              echo "Registering deploy key..."
+              echo "Scanning Forgejo SSH host key..."
-              HTTP_CODE=$(curl -sk -o /tmp/response.json -w "%{http_code}" \
+              HOSTKEY=$(ssh-keyscan -p "${FORGEJO_SSH_PORT}" "${FORGEJO_HOST}" 2>/dev/null | grep -v '^#' | head -1)
-                -X POST "${FORGEJO_URL}/api/v1/repos/${REPO_OWNER}/${REPO_NAME}/keys" \
+              if [ -z "${HOSTKEY}" ]; then
-                -H "Content-Type: application/json" \
+                echo "Failed to scan SSH host key"
-                -u "${ADMIN_USER}:${ADMIN_PASS}" \
-                -d "{\"title\":\"argocd-deploy-key\",\"key\":\"${PUBKEY}\",\"read_only\":true}")
-
-              if [ "${HTTP_CODE}" = "201" ]; then
-                echo "Deploy key registered successfully"
-              elif [ "${HTTP_CODE}" = "422" ]; then
-                echo "Deploy key already exists in Forgejo (422), continuing"
-              else
-                echo "Failed to register deploy key: HTTP ${HTTP_CODE}"
-                cat /tmp/response.json
                 exit 1
               fi
+              echo "Got host key: ${HOSTKEY}"
 
-              # Create ArgoCD repository secret
+              EXISTING=$(kubectl get configmap "${KNOWN_HOSTS_CM}" -n "${ARGOCD_NS}" -o jsonpath='{.data.ssh_known_hosts}')
-              kubectl create secret generic "${REPO_SECRET}" \
+              if echo "${EXISTING}" | grep -qF "[${FORGEJO_HOST}]:${FORGEJO_SSH_PORT}"; then
-                -n "${ARGOCD_NS}" \
+                echo "Forgejo host key already in known hosts"
-                --from-literal=type=git \
+              else
-                --from-literal=url="${REPO_URL}" \
+                echo "Adding Forgejo host key to ArgoCD known hosts..."
-                --from-literal=sshPrivateKey="${PRIVKEY}"
+                UPDATED=$(printf '%s\n%s\n' "${EXISTING}" "${HOSTKEY}")
+                kubectl patch configmap "${KNOWN_HOSTS_CM}" -n "${ARGOCD_NS}" \
+                  --type merge -p "{\"data\":{\"ssh_known_hosts\":$(printf '%s' "${UPDATED}" | jq -Rs .)}}"
+                echo "Added Forgejo SSH host key to known hosts"
+              fi
 
-              # Label the secret for ArgoCD
+              # Step 2: Create deploy key and repo secret (if not already done)
-              kubectl label secret "${REPO_SECRET}" \
+              if kubectl get secret "${REPO_SECRET}" -n "${ARGOCD_NS}" >/dev/null 2>&1; then
-                -n "${ARGOCD_NS}" \
+                echo "Secret ${REPO_SECRET} already exists in ${ARGOCD_NS}, skipping key generation"
-                argocd.argoproj.io/secret-type=repository
+              else
+                # Read admin credentials
+                ADMIN_USER=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.username}' | base64 -d)
+                ADMIN_PASS=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.password}' | base64 -d)
 
-              echo "Created ArgoCD repository secret ${REPO_SECRET} in ${ARGOCD_NS}"
+                # Generate ed25519 SSH keypair
+                KEYDIR=$(mktemp -d)
+                ssh-keygen -t ed25519 -f "${KEYDIR}/id_ed25519" -N "" -q
+                PRIVKEY=$(cat "${KEYDIR}/id_ed25519")
+                PUBKEY=$(cat "${KEYDIR}/id_ed25519.pub")
+                rm -rf "${KEYDIR}"
+
+                # Register deploy key via Forgejo API
+                echo "Registering deploy key..."
+                HTTP_CODE=$(curl -sk -o /tmp/response.json -w "%{http_code}" \
+                  -X POST "${FORGEJO_URL}/api/v1/repos/${REPO_OWNER}/${REPO_NAME}/keys" \
+                  -H "Content-Type: application/json" \
+                  -u "${ADMIN_USER}:${ADMIN_PASS}" \
+                  -d "{\"title\":\"argocd-deploy-key\",\"key\":\"${PUBKEY}\",\"read_only\":true}")
+
+                if [ "${HTTP_CODE}" = "201" ]; then
+                  echo "Deploy key registered successfully"
+                elif [ "${HTTP_CODE}" = "422" ]; then
+                  echo "Deploy key already exists in Forgejo (422), continuing"
+                else
+                  echo "Failed to register deploy key: HTTP ${HTTP_CODE}"
+                  cat /tmp/response.json
+                  exit 1
+                fi
+
+                # Create ArgoCD repository secret with label
+                cat <<EOSECRET | kubectl apply -f -
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: ${REPO_SECRET}
+                namespace: ${ARGOCD_NS}
+                labels:
+                  argocd.argoproj.io/secret-type: repository
+              stringData:
+                type: git
+                url: "${REPO_URL}"
+                sshPrivateKey: |
+              $(echo "${PRIVKEY}" | sed 's/^/            /')
+              EOSECRET
+
+                echo "Created ArgoCD repository secret ${REPO_SECRET} in ${ARGOCD_NS}"
+              fi
+
+              echo "Done"
           env:
             - name: NAMESPACE
               valueFrom:

rendered/envs/production/forgejo/clusterrole-argocd-deploy-key-init.yaml

@@ -13,3 +13,10 @@ rules:
     verbs:
       - get
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - patch

rendered/envs/production/forgejo/job-argocd-deploy-key-init.yaml

@@ -15,37 +15,23 @@ spec:
             - |
               set -e
 
-              apk add --no-cache openssh-keygen > /dev/null 2>&1
+              apk add --no-cache openssh-keygen openssh-client > /dev/null 2>&1
 
               ARGOCD_NS="argocd"
               REPO_SECRET="forgejo-repo"
               REPO_URL="ssh://git@git.tr1ceracop.de:222/gitea_admin/k8s-and-chill.git"
-              FORGEJO_URL="https://git.tr1ceracop.de"
+              FORGEJO_HOST="git.tr1ceracop.de"
+              FORGEJO_SSH_PORT="222"
+              FORGEJO_URL="https://${FORGEJO_HOST}"
               REPO_OWNER="gitea_admin"
               REPO_NAME="k8s-and-chill"
-
+              KNOWN_HOSTS_CM="argocd-ssh-known-hosts-cm"
-              # Check if ArgoCD repo secret already exists
-              if kubectl get secret "${REPO_SECRET}" -n "${ARGOCD_NS}" >/dev/null 2>&1; then
-                echo "Secret ${REPO_SECRET} already exists in ${ARGOCD_NS}, skipping"
-                exit 0
-              fi
-
-              # Read admin credentials from forgejo-admin-secret
-              ADMIN_USER=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.username}' | base64 -d)
-              ADMIN_PASS=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.password}' | base64 -d)
-
-              # Generate ed25519 SSH keypair
-              KEYDIR=$(mktemp -d)
-              ssh-keygen -t ed25519 -f "${KEYDIR}/id_ed25519" -N "" -q
-              PRIVKEY=$(cat "${KEYDIR}/id_ed25519")
-              PUBKEY=$(cat "${KEYDIR}/id_ed25519.pub")
-              rm -rf "${KEYDIR}"
 
               # Wait for Forgejo to be ready
               echo "Waiting for Forgejo to be ready..."
               for i in $(seq 1 60); do
                 if curl -sk "${FORGEJO_URL}/api/v1/version" >/dev/null 2>&1; then
-                  echo "Forgejo is ready"
+                  echo "Forgejo HTTPS is ready"
                   break
                 fi
                 if [ "$i" -eq 60 ]; then
@@ -55,37 +41,79 @@ spec:
                 sleep 5
               done
 
-              # Register deploy key via Forgejo API
+              # Step 1: Add Forgejo SSH host key to ArgoCD known hosts
-              echo "Registering deploy key..."
+              echo "Scanning Forgejo SSH host key..."
-              HTTP_CODE=$(curl -sk -o /tmp/response.json -w "%{http_code}" \
+              HOSTKEY=$(ssh-keyscan -p "${FORGEJO_SSH_PORT}" "${FORGEJO_HOST}" 2>/dev/null | grep -v '^#' | head -1)
-                -X POST "${FORGEJO_URL}/api/v1/repos/${REPO_OWNER}/${REPO_NAME}/keys" \
+              if [ -z "${HOSTKEY}" ]; then
-                -H "Content-Type: application/json" \
+                echo "Failed to scan SSH host key"
-                -u "${ADMIN_USER}:${ADMIN_PASS}" \
-                -d "{\"title\":\"argocd-deploy-key\",\"key\":\"${PUBKEY}\",\"read_only\":true}")
-
-              if [ "${HTTP_CODE}" = "201" ]; then
-                echo "Deploy key registered successfully"
-              elif [ "${HTTP_CODE}" = "422" ]; then
-                echo "Deploy key already exists in Forgejo (422), continuing"
-              else
-                echo "Failed to register deploy key: HTTP ${HTTP_CODE}"
-                cat /tmp/response.json
                 exit 1
               fi
+              echo "Got host key: ${HOSTKEY}"
 
-              # Create ArgoCD repository secret
+              EXISTING=$(kubectl get configmap "${KNOWN_HOSTS_CM}" -n "${ARGOCD_NS}" -o jsonpath='{.data.ssh_known_hosts}')
-              kubectl create secret generic "${REPO_SECRET}" \
+              if echo "${EXISTING}" | grep -qF "[${FORGEJO_HOST}]:${FORGEJO_SSH_PORT}"; then
-                -n "${ARGOCD_NS}" \
+                echo "Forgejo host key already in known hosts"
-                --from-literal=type=git \
+              else
-                --from-literal=url="${REPO_URL}" \
+                echo "Adding Forgejo host key to ArgoCD known hosts..."
-                --from-literal=sshPrivateKey="${PRIVKEY}"
+                UPDATED=$(printf '%s\n%s\n' "${EXISTING}" "${HOSTKEY}")
+                kubectl patch configmap "${KNOWN_HOSTS_CM}" -n "${ARGOCD_NS}" \
+                  --type merge -p "{\"data\":{\"ssh_known_hosts\":$(printf '%s' "${UPDATED}" | jq -Rs .)}}"
+                echo "Added Forgejo SSH host key to known hosts"
+              fi
 
-              # Label the secret for ArgoCD
+              # Step 2: Create deploy key and repo secret (if not already done)
-              kubectl label secret "${REPO_SECRET}" \
+              if kubectl get secret "${REPO_SECRET}" -n "${ARGOCD_NS}" >/dev/null 2>&1; then
-                -n "${ARGOCD_NS}" \
+                echo "Secret ${REPO_SECRET} already exists in ${ARGOCD_NS}, skipping key generation"
-                argocd.argoproj.io/secret-type=repository
+              else
+                # Read admin credentials
+                ADMIN_USER=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.username}' | base64 -d)
+                ADMIN_PASS=$(kubectl get secret forgejo-admin-secret -n "${NAMESPACE}" -o jsonpath='{.data.password}' | base64 -d)
 
-              echo "Created ArgoCD repository secret ${REPO_SECRET} in ${ARGOCD_NS}"
+                # Generate ed25519 SSH keypair
+                KEYDIR=$(mktemp -d)
+                ssh-keygen -t ed25519 -f "${KEYDIR}/id_ed25519" -N "" -q
+                PRIVKEY=$(cat "${KEYDIR}/id_ed25519")
+                PUBKEY=$(cat "${KEYDIR}/id_ed25519.pub")
+                rm -rf "${KEYDIR}"
+
+                # Register deploy key via Forgejo API
+                echo "Registering deploy key..."
+                HTTP_CODE=$(curl -sk -o /tmp/response.json -w "%{http_code}" \
+                  -X POST "${FORGEJO_URL}/api/v1/repos/${REPO_OWNER}/${REPO_NAME}/keys" \
+                  -H "Content-Type: application/json" \
+                  -u "${ADMIN_USER}:${ADMIN_PASS}" \
+                  -d "{\"title\":\"argocd-deploy-key\",\"key\":\"${PUBKEY}\",\"read_only\":true}")
+
+                if [ "${HTTP_CODE}" = "201" ]; then
+                  echo "Deploy key registered successfully"
+                elif [ "${HTTP_CODE}" = "422" ]; then
+                  echo "Deploy key already exists in Forgejo (422), continuing"
+                else
+                  echo "Failed to register deploy key: HTTP ${HTTP_CODE}"
+                  cat /tmp/response.json
+                  exit 1
+                fi
+
+                # Create ArgoCD repository secret with label
+                cat <<EOSECRET | kubectl apply -f -
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: ${REPO_SECRET}
+                namespace: ${ARGOCD_NS}
+                labels:
+                  argocd.argoproj.io/secret-type: repository
+              stringData:
+                type: git
+                url: "${REPO_URL}"
+                sshPrivateKey: |
+              $(echo "${PRIVKEY}" | sed 's/^/            /')
+              EOSECRET
+
+                echo "Created ArgoCD repository secret ${REPO_SECRET} in ${ARGOCD_NS}"
+              fi
+
+              echo "Done"
           env:
             - name: NAMESPACE
               valueFrom: