merge: reconcile with GitLab CI fixes landed concurrently

.gitlab-ci.yml

@@ -1,80 +1,30 @@
 # GitLab-CI dev→prod pipeline for VNCmail+ — GitOps via ArgoCD.
 #
-# Revised after direct inspection of the real infrastructure found ArgoCD
-# already installed (idle, zero Applications) on the dev-k8s-1/2/3 cluster.
-# That's more idiomatic than a runner-executes-kubectl design, and it means
-# this pipeline needs ZERO cluster credentials — CI only ever talks to the
-# container registry and to this git repo. ArgoCD (which already has
-# whatever cluster access it needs, set up once when its Applications were
-# registered — see deploy/argocd/) is what actually applies anything.
-#
 # Design:
-#   - One image name, environment lives only in the tag. No more -dev/-beta
-#     name confusion from the old GitHub Actions workflow.
 #   - MR into `dev`: verify only (typecheck/lint/unit test/build check). No
 #     push, no deploy — this is the multi-developer merge gate.
-#   - Push to `dev`: build+push an immutable `sha-<sha>` tag, then commit a
+#   - Push to `dev`: build+push an immutable `sha-<sha>` tag with Docker +
-#     one-line tag-bump into overlays/dev/image-tag/kustomization.yaml
+#     docker-in-docker, then commit a one-line tag-bump into
-#     (`[skip ci]`, so this doesn't retrigger itself). ArgoCD's `vncmail-dev`
+#     overlays/dev/image-tag/kustomization.yaml (`[skip ci]`). ArgoCD's
-#     Application has automated sync — it notices the git change and applies
+#     `vncmail-dev` Application syncs it automatically.
-#     it. No approval needed, dev always deploys, and this job never touches
+#   - Push to `main`: NEVER rebuilds. `main` only advances via
-#     the cluster directly.
-#   - Push to `main`: NEVER rebuilds. `main` only ever advances via
 #     `git merge --ff-only dev`, so main's HEAD commit already has a built
-#     image (the same sha- tag dev already deployed). This job just bumps
+#     image. This job just bumps overlays/prod/image-tag/kustomization.yaml
-#     overlays/prod/image-tag/kustomization.yaml to point at that same tag.
+#     to point at that same tag. The actual promotion gate is a HUMAN
-#     The actual promotion gate is a HUMAN clicking Sync on the `vncmail-prod` ArgoCD
+#     clicking Sync on the `vncmail-prod` ArgoCD Application.
-#     Application (deliberately NOT automated sync) — not a GitLab manual
-#     job, since ArgoCD already provides that exact gate more directly.
-#     Until prod Stalwart/hostname/secrets are real (see VNCMAIL-SETUP.md),
-#     nobody should click that Sync button — but nothing here does it for
-#     you either way.
 #
 # Deliberately single-platform (linux/amd64) — this pipeline serves two
 # known amd64 microk8s clusters, not public multi-arch distribution (that's
 # what the GHCR release workflows are for, untouched by this file).
 #
-# Registry history (so nobody re-litigates this from scratch). GitLab's own
+# Prerequisite this file assumes:
-# Container Registry was tried twice and does not work on this server:
+#   - A GitLab Runner with Docker-in-Docker service support (Kubernetes or
-#
+#     Docker executor). The `docker:28.4.0-dind` service requires privileged
-#   Round 1: registry_external_url was unset, so $CI_REGISTRY was empty and
+#     mode on most Kubernetes executors.
-#            docker login silently fell through to Docker Hub.
-#   Round 2: after the server-side config, the project's sidebar DID show
-#            "Container Registry" and docker login DID succeed - but the push
-#            failed 403. Diagnosed 2026-08-05 by curling the vhost directly:
-#
-#              $ curl -i https://registry.gitlab.vnc.biz/v2/
-#              www-authenticate: Bearer realm="http://gitlab.vnc.biz/jwt/auth",
-#                                       service="dependency_proxy"
-#              x-runtime: 0.020470
-#              x-gitlab-meta: {"correlation_id":...}
-#
-#            x-runtime/x-gitlab-meta are RAILS headers, and the service is
-#            "dependency_proxy" - so nginx routes that hostname to the GitLab
-#            Rails app, which reads /v2/ as the dependency proxy (a Docker Hub
-#            pull-through cache), NOT to the registry container. The registry
-#            service was never actually wired behind that vhost. That is why
-#            login worked (Rails issues an unscoped dependency-proxy token)
-#            while a scoped :push request 403'd - the dependency proxy has no
-#            push concept at all.
-#
-# Fixing that is an nginx/omnibus change on the GitLab server (registry
-# service must actually listen behind registry.gitlab.vnc.biz), not something
-# any .gitlab-ci.yml can reach. Until someone does that, GHCR it is - the
-# same image the sandbox already pulls, and confirmed public so the cluster
-# needs no imagePullSecrets (see deploy/k8s/base/deployment.yaml).
-#
-# Prerequisite this file assumes (documented in VNCMAIL-SETUP.md, not
-# something this file can set up itself):
-#   - A GitHub PAT with `write:packages` for ghcr.io/brvncde-dotcom as
-#     $GITLAB_CI_GHCR_TOKEN, plus the matching GitHub username as
-#     $GITLAB_CI_GHCR_USER — both masked+protected CI/CD variables. A GitHub
-#     credential can only come from GitHub; nothing GitLab-native substitutes.
-#   - A GitLab Runner (any kind — no cluster access needed at all now).
 #   - Either "allow this job token to push to this project" enabled
 #     (Settings → CI/CD → Job token permissions), OR a project access token
-#     with `write_repository` scope in $GITLAB_PUSH_TOKEN. The job below
+#     with `write_repository` scope in $GITLAB_PUSH_TOKEN. The bump jobs
-#     tries CI_JOB_TOKEN first (see the script).
+#     try CI_JOB_TOKEN first (see the script).
 #
 # deploy/k8s/ca/ (the EJBCA internal CA) is never referenced anywhere below,
 # and neither ArgoCD Application in deploy/argocd/ points at it — that stays
@@ -87,11 +37,14 @@ stages:
   - bump-prod
 
 variables:
-  # GHCR, not GitLab's own registry - see the "Registry history" note in the
+  IMAGE: $CI_REGISTRY_IMAGE
-  # header for the curl output proving why. Same image the sandbox already
-  # pulls today.
-  IMAGE: ghcr.io/brvncde-dotcom/vncmail-plus-dev
   GIT_STRATEGY: clone
+  DOCKER_DRIVER: overlay2
+  # DinD service is reached at the `docker` alias (set explicitly on the
+  # service below), not localhost. TLS disabled so the daemon listens on
+  # plaintext 2375 — same pattern as the working vnc-localidp pipeline.
+  DOCKER_HOST: tcp://docker:2375
+  DOCKER_TLS_CERTDIR: ""
 
 # ---------------------------------------------------------------------------
 # verify — required check on every MR into dev. No registry, no cluster.
@@ -118,45 +71,24 @@ verify:
 # ---------------------------------------------------------------------------
 build:
   stage: build
-  # Kaniko builds OCI images without a Docker daemon, so it needs neither a
+  image: docker:28.4.0
-  # dind service nor a privileged pod — GitLab's own recommended approach
+  services:
-  # for the Kubernetes executor specifically. The docker:27-cli + dind
+    - name: docker:28.4.0-dind
-  # combination that was here before this got as far as a successful
+      alias: docker
-  # $CI_REGISTRY login, then failed every way it was pointed
-  # (unix:///var/run/docker.sock, tcp://docker:2375, tcp://localhost:2375):
-  # the dind container itself was never actually listening, which on this
-  # executor means it needs `privileged: true` in the runner's own
-  # config.toml — a cluster/GitLab-admin setting outside this file's
-  # control. Kaniko sidesteps that requirement entirely rather than chasing
-  # runner permissions further, and is also the safer default on a shared
-  # cluster (no privileged containers at all).
-  image:
-    name: gcr.io/kaniko-project/executor:v1.23.2-debug
-    entrypoint: [""]
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
+  before_script:
+    - until docker info; do sleep 1; done
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
   script:
-    # Fail loudly and immediately if the GHCR credentials aren't configured,
-    # rather than letting kaniko get all the way through a full Next.js build
-    # and only then 403 on push (which is exactly how the GitLab-registry
-    # attempt burned several pipeline runs).
-    - |
-      if [ -z "$GITLAB_CI_GHCR_TOKEN" ] || [ -z "$GITLAB_CI_GHCR_USER" ]; then
-        echo "ERROR: \$GITLAB_CI_GHCR_TOKEN and/or \$GITLAB_CI_GHCR_USER are not set."
-        echo "Add both under Settings -> CI/CD -> Variables (masked + protected)."
-        echo "The token is a GitHub PAT with the write:packages scope."
-        exit 1
-      fi
-    - mkdir -p /kaniko/.docker
-    - |
-      echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"$(printf '%s:%s' "$GITLAB_CI_GHCR_USER" "$GITLAB_CI_GHCR_TOKEN" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
     - >
-      /kaniko/executor
+      docker build
-      --context "$CI_PROJECT_DIR"
-      --dockerfile "$CI_PROJECT_DIR/Dockerfile"
       --build-arg GIT_COMMIT=$CI_COMMIT_SHA
-      --destination "$IMAGE:sha-$CI_COMMIT_SHORT_SHA"
+      -t "$IMAGE:sha-$CI_COMMIT_SHORT_SHA"
-      --destination "$IMAGE:dev-latest"
+      -t "$IMAGE:dev-latest"
+      .
+    - docker push "$IMAGE:sha-$CI_COMMIT_SHORT_SHA"
+    - docker push "$IMAGE:dev-latest"
 
 # ---------------------------------------------------------------------------
 # bump-dev — no cluster access. Commits the just-built tag into the overlay
@@ -164,7 +96,9 @@ build:
 # ---------------------------------------------------------------------------
 bump-dev:
   stage: bump-dev
-  image: alpine/git:2.47.0
+  # alpine/git:2.47.0 was never published on Docker Hub — the 2.47.x line
+  # starts at 2.47.1. Using 2.47.2 (latest 2.47.x).
+  image: alpine/git:2.47.2
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
   script:
@@ -176,7 +110,7 @@ bump-dev:
       apiVersion: kustomize.config.k8s.io/v1alpha1
       kind: Component
       images:
-        - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
+        - name: vncmail-plus
           newName: $IMAGE
           newTag: $TAG
       EOF
@@ -199,7 +133,7 @@ bump-dev:
 # ---------------------------------------------------------------------------
 bump-prod:
   stage: bump-prod
-  image: alpine/git:2.47.0
+  image: alpine/git:2.47.2
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"'
   script:
@@ -215,7 +149,7 @@ bump-prod:
       apiVersion: kustomize.config.k8s.io/v1alpha1
       kind: Component
       images:
-        - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
+        - name: vncmail-plus
           newName: $IMAGE
           newTag: $TAG
       EOF

deploy/k8s/README.md

@@ -59,10 +59,9 @@ is CI's job to decide — it's an explicit, human-triggered event.
 | 7 | Ingress `vncmail-plus` | `base/ingress.yaml` (+ overlay patches for prod) | TLS host |
 
 **Image:** CI builds and pushes to `registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus`
-(tag `sha-<sha>` per deploy, moving pointers `dev-latest`/`prod-latest`). The
+(tag `sha-<sha>` per deploy, moving pointer `dev-latest`). The generic `vncmail-plus`
-`ghcr.io/brvncde-dotcom/vncmail-plus-dev` image referenced in `base/deployment.yaml`
+image name in `base/deployment.yaml` is a placeholder — kustomize's image-tag
-is a legacy default only — CI overrides it per-deploy via `kubectl set image`,
+Component replaces it with the real registry path on every deploy.
-so what's committed there never needs to track what's actually running.
 
 ---
 
@@ -98,15 +97,15 @@ pick up the fix):
 ```bash
 cd deploy/k8s/overlays/dev     # or overlays/prod, once real
 
-# a) Image-pull secret — the registry package is private.
+# a) Image-pull secret — the GitLab registry requires authentication.
-kubectl create secret docker-registry ghcr-pull \
+#    Use a project deploy token with `read_registry` scope, or the CI job
+#    token (short-lived — better for CI, not for long-running clusters).
+kubectl create secret docker-registry gitlab-registry \
   --namespace vncmail \
-  --docker-server=ghcr.io \
+  --docker-server=registry.gitlab.vnc.biz \
-  --docker-username=brvncde-dotcom \
+  --docker-username=<deploy-token-name> \
-  --docker-password='<GITHUB_PAT_read:packages>' \
+  --docker-password='<deploy-token-secret>' \
-  --docker-email=br@vnc.biz
+  --docker-email=ci@vnc.biz
-#    Once CI has cut over to registry.gitlab.vnc.biz, this becomes a
-#    docker-registry secret for that registry instead — see VNCMAIL-SETUP.md.
 
 # b) App config secret — copy the template, set a real SESSION_SECRET, apply.
 cp secret.example.yaml secret.yaml
@@ -117,8 +116,8 @@ kubectl apply -f secret.yaml
 kubectl apply -k .
 ```
 
-> Alternative to (a): make the registry package public, then delete the
+> Alternative to (a): make the GitLab container registry public for this
-> `imagePullSecrets:` block from `base/deployment.yaml`.
+> project, then delete the `imagePullSecrets:` block from `base/deployment.yaml`.
 
 After this one-time setup, routine deploys to `dev` happen automatically via
 CI on every push — see "Routine deploys go through CI now" above. This
@@ -170,7 +169,7 @@ ArgoCD re-sync.
 
 | Symptom | Cause / fix |
 |---------|-------------|
-| Pod `ImagePullBackOff` | `ghcr-pull` secret missing/expired, or package still private. Recreate the secret (§3a) or make the package public. |
+| Pod `ImagePullBackOff` | `gitlab-registry` secret missing/expired, or token lacks `read_registry`. Recreate the secret (§3a) or make the registry public. |
 | Pod `CrashLoopBackOff`, logs show `EACCES`/permission on `/app/data` | Volume not writable by uid 1001. `securityContext.fsGroup: 1001` is set in `base/deployment.yaml` — keep it; some storage drivers also need it on the PVC. |
 | PVC stuck `Pending` | Wrong `storageClassName` in `base/pvc.yaml`. Set it to one from `kubectl get sc`. |
 | Ingress has no address / no cert | Wrong `ingressClassName` or cert issuer. Match bulwark's (§2). Check `kubectl -n vncmail describe ingress vncmail-plus`. |

deploy/k8s/base/deployment.yaml

@@ -23,20 +23,18 @@ spec:
         fsGroup: 1001
         runAsUser: 1001
         runAsGroup: 1001
-      # Confirmed 2026-08-05: ghcr.io/brvncde-dotcom/vncmail-plus-dev IS public
+      # The GitLab container registry is private by default. Nodes need a
-      # (anonymous token pull succeeded) — no imagePullSecrets needed. This is
+      # docker-registry secret named `gitlab-registry` in the target namespace.
-      # deploy/k8s/README.md's own documented alternative to creating a
+      # Create it once per environment during first-time setup
-      # ghcr-pull secret. Removed rather than left referencing a
+      # (see deploy/k8s/README.md §3a).
-      # not-yet-created secret, which would otherwise block every pod from
+      imagePullSecrets:
-      # starting regardless of the image being public (kubelet fails to
+        - name: gitlab-registry
-      # resolve a missing imagePullSecrets entry before it ever gets to
-      # deciding whether auth was actually required).
       containers:
         - name: vncmail-plus
-          # Default/legacy value — CI overrides the image per-deploy via
+          # Generic placeholder — the real image name + tag are injected by the
-          # `kustomize edit set image`, so what's committed here never goes
+          # image-tag kustomize Component on every deploy (see
-          # stale. For a one-off manual apply, pin a digest instead of :latest.
+          # overlays/*/image-tag/kustomization.yaml, rewritten by CI).
-          image: ghcr.io/brvncde-dotcom/vncmail-plus-dev:latest
+          image: vncmail-plus:latest
           imagePullPolicy: Always
           ports:
             - containerPort: 3000

deploy/k8s/base/ingress.yaml

@@ -15,6 +15,7 @@ metadata:
   name: vncmail-plus
   annotations:
     cert-manager.io/cluster-issuer: CHANGEME
+    traefik.ingress.kubernetes.io/router.middlewares: traefik-redirect-to-https@kubernetescrd
 spec:
   ingressClassName: traefik
   tls:

deploy/k8s/overlays/dev/image-tag/kustomization.yaml

@@ -1,11 +1,8 @@
-# Owned by CI (the bump-dev job in .gitlab-ci.yml), not by hand. Kept as its
+# Owned by CI (bump-dev job in .gitlab-ci.yml) - regenerated every
-# own Component so CI only ever rewrites this 6-line file, never the parent
+# push to dev. Do not hand-edit; edits here get overwritten.
-# overlays/dev/kustomization.yaml (structure/patches there stay under normal
-# code review — CI regenerating a whole hand-maintained file on every push
-# would silently revert any change made there between deploys).
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
 images:
-  - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
+  - name: vncmail-plus
-    newName: ghcr.io/brvncde-dotcom/vncmail-plus-dev
+    newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus
-    newTag: sha-147660a
+    newTag: sha-f46614a4

deploy/k8s/overlays/dev/patch-image-pull-policy.yaml

@@ -4,19 +4,6 @@
 # is pure waste - the content behind that tag can never change, so re-pulling
 # it on every pod start only adds a registry round-trip and a hard dependency
 # on the registry being reachable at scheduling time.
-#
-# It is also load-bearing right now: until CI can actually push (GitLab's
-# registry vhost serves Rails, not the registry - see .gitlab-ci.yml's
-# "Registry history" note), sha- tagged images are side-loaded straight into
-# each node's containerd:
-#
-#   docker save --platform linux/amd64 -o vncmail.tar <image>:<tag>
-#   scp vncmail.tar dev-k8s-N:/tmp/ && ssh dev-k8s-N \
-#     'microk8s ctr images import /tmp/vncmail.tar'
-#
-# imported to ALL of dev-k8s-1/2/3 so the pod can schedule anywhere. With
-# Always, kubelet would ignore that local image and fail on a registry pull
-# for a tag the registry has never seen.
 apiVersion: apps/v1
 kind: Deployment
 metadata:

deploy/k8s/overlays/dev/patch-ingress.yaml

@@ -1,14 +1,12 @@
-# dev-k8s cluster confirmed to have a `letsencrypt-staging` ClusterIssuer
+# Dev now uses letsencrypt-prod so the sandbox certificate is trusted by
-# already (no `letsencrypt-prod` exists there) - staging avoids burning
+# browsers. Both letsencrypt-staging and letsencrypt-prod ClusterIssuers
-# Let's Encrypt's real rate limits while this is still being stood up.
+# exist on the dev-k8s cluster (see VNCiAC-dev-cluster-runbook.md §7).
-# vncmail.sandbox.vnc.de DNS does not point here yet either - this is the
-# intended host, not a live one (see VNCMAIL-SETUP.md for what's still open).
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: vncmail-plus
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-staging
+    cert-manager.io/cluster-issuer: letsencrypt-prod
 spec:
   tls:
     - hosts:

deploy/k8s/overlays/prod/image-tag/kustomization.yaml

@@ -7,6 +7,6 @@
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
 images:
-  - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
+  - name: vncmail-plus
     newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus
     newTag: not-yet-promoted