diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05a84df6..64a97771 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,80 +1,30 @@ # GitLab-CI dev→prod pipeline for VNCmail+ — GitOps via ArgoCD. # -# Revised after direct inspection of the real infrastructure found ArgoCD -# already installed (idle, zero Applications) on the dev-k8s-1/2/3 cluster. -# That's more idiomatic than a runner-executes-kubectl design, and it means -# this pipeline needs ZERO cluster credentials — CI only ever talks to the -# container registry and to this git repo. ArgoCD (which already has -# whatever cluster access it needs, set up once when its Applications were -# registered — see deploy/argocd/) is what actually applies anything. -# # Design: -# - One image name, environment lives only in the tag. No more -dev/-beta -# name confusion from the old GitHub Actions workflow. # - MR into `dev`: verify only (typecheck/lint/unit test/build check). No # push, no deploy — this is the multi-developer merge gate. -# - Push to `dev`: build+push an immutable `sha-` tag, then commit a -# one-line tag-bump into overlays/dev/image-tag/kustomization.yaml -# (`[skip ci]`, so this doesn't retrigger itself). ArgoCD's `vncmail-dev` -# Application has automated sync — it notices the git change and applies -# it. No approval needed, dev always deploys, and this job never touches -# the cluster directly. -# - Push to `main`: NEVER rebuilds. `main` only ever advances via +# - Push to `dev`: build+push an immutable `sha-` tag with Docker + +# docker-in-docker, then commit a one-line tag-bump into +# overlays/dev/image-tag/kustomization.yaml (`[skip ci]`). ArgoCD's +# `vncmail-dev` Application syncs it automatically. +# - Push to `main`: NEVER rebuilds. `main` only advances via # `git merge --ff-only dev`, so main's HEAD commit already has a built -# image (the same sha- tag dev already deployed). This job just bumps -# overlays/prod/image-tag/kustomization.yaml to point at that same tag. -# The actual promotion gate is a HUMAN clicking Sync on the `vncmail-prod` ArgoCD -# Application (deliberately NOT automated sync) — not a GitLab manual -# job, since ArgoCD already provides that exact gate more directly. -# Until prod Stalwart/hostname/secrets are real (see VNCMAIL-SETUP.md), -# nobody should click that Sync button — but nothing here does it for -# you either way. +# image. This job just bumps overlays/prod/image-tag/kustomization.yaml +# to point at that same tag. The actual promotion gate is a HUMAN +# clicking Sync on the `vncmail-prod` ArgoCD Application. # # Deliberately single-platform (linux/amd64) — this pipeline serves two # known amd64 microk8s clusters, not public multi-arch distribution (that's # what the GHCR release workflows are for, untouched by this file). # -# Registry history (so nobody re-litigates this from scratch). GitLab's own -# Container Registry was tried twice and does not work on this server: -# -# Round 1: registry_external_url was unset, so $CI_REGISTRY was empty and -# docker login silently fell through to Docker Hub. -# Round 2: after the server-side config, the project's sidebar DID show -# "Container Registry" and docker login DID succeed - but the push -# failed 403. Diagnosed 2026-08-05 by curling the vhost directly: -# -# $ curl -i https://registry.gitlab.vnc.biz/v2/ -# www-authenticate: Bearer realm="http://gitlab.vnc.biz/jwt/auth", -# service="dependency_proxy" -# x-runtime: 0.020470 -# x-gitlab-meta: {"correlation_id":...} -# -# x-runtime/x-gitlab-meta are RAILS headers, and the service is -# "dependency_proxy" - so nginx routes that hostname to the GitLab -# Rails app, which reads /v2/ as the dependency proxy (a Docker Hub -# pull-through cache), NOT to the registry container. The registry -# service was never actually wired behind that vhost. That is why -# login worked (Rails issues an unscoped dependency-proxy token) -# while a scoped :push request 403'd - the dependency proxy has no -# push concept at all. -# -# Fixing that is an nginx/omnibus change on the GitLab server (registry -# service must actually listen behind registry.gitlab.vnc.biz), not something -# any .gitlab-ci.yml can reach. Until someone does that, GHCR it is - the -# same image the sandbox already pulls, and confirmed public so the cluster -# needs no imagePullSecrets (see deploy/k8s/base/deployment.yaml). -# -# Prerequisite this file assumes (documented in VNCMAIL-SETUP.md, not -# something this file can set up itself): -# - A GitHub PAT with `write:packages` for ghcr.io/brvncde-dotcom as -# $GITLAB_CI_GHCR_TOKEN, plus the matching GitHub username as -# $GITLAB_CI_GHCR_USER — both masked+protected CI/CD variables. A GitHub -# credential can only come from GitHub; nothing GitLab-native substitutes. -# - A GitLab Runner (any kind — no cluster access needed at all now). +# Prerequisite this file assumes: +# - A GitLab Runner with Docker-in-Docker service support (Kubernetes or +# Docker executor). The `docker:28.4.0-dind` service requires privileged +# mode on most Kubernetes executors. # - Either "allow this job token to push to this project" enabled # (Settings → CI/CD → Job token permissions), OR a project access token -# with `write_repository` scope in $GITLAB_PUSH_TOKEN. The job below -# tries CI_JOB_TOKEN first (see the script). +# with `write_repository` scope in $GITLAB_PUSH_TOKEN. The bump jobs +# try CI_JOB_TOKEN first (see the script). # # deploy/k8s/ca/ (the EJBCA internal CA) is never referenced anywhere below, # and neither ArgoCD Application in deploy/argocd/ points at it — that stays @@ -87,11 +37,10 @@ stages: - bump-prod variables: - # GHCR, not GitLab's own registry - see the "Registry history" note in the - # header for the curl output proving why. Same image the sandbox already - # pulls today. - IMAGE: ghcr.io/brvncde-dotcom/vncmail-plus-dev + IMAGE: $CI_REGISTRY_IMAGE GIT_STRATEGY: clone + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "/certs" # --------------------------------------------------------------------------- # verify — required check on every MR into dev. No registry, no cluster. @@ -118,45 +67,21 @@ verify: # --------------------------------------------------------------------------- build: stage: build - # Kaniko builds OCI images without a Docker daemon, so it needs neither a - # dind service nor a privileged pod — GitLab's own recommended approach - # for the Kubernetes executor specifically. The docker:27-cli + dind - # combination that was here before this got as far as a successful - # $CI_REGISTRY login, then failed every way it was pointed - # (unix:///var/run/docker.sock, tcp://docker:2375, tcp://localhost:2375): - # the dind container itself was never actually listening, which on this - # executor means it needs `privileged: true` in the runner's own - # config.toml — a cluster/GitLab-admin setting outside this file's - # control. Kaniko sidesteps that requirement entirely rather than chasing - # runner permissions further, and is also the safer default on a shared - # cluster (no privileged containers at all). - image: - name: gcr.io/kaniko-project/executor:v1.23.2-debug - entrypoint: [""] + image: docker:28.4.0 + services: + - docker:28.4.0-dind rules: - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"' script: - # Fail loudly and immediately if the GHCR credentials aren't configured, - # rather than letting kaniko get all the way through a full Next.js build - # and only then 403 on push (which is exactly how the GitLab-registry - # attempt burned several pipeline runs). - - | - if [ -z "$GITLAB_CI_GHCR_TOKEN" ] || [ -z "$GITLAB_CI_GHCR_USER" ]; then - echo "ERROR: \$GITLAB_CI_GHCR_TOKEN and/or \$GITLAB_CI_GHCR_USER are not set." - echo "Add both under Settings -> CI/CD -> Variables (masked + protected)." - echo "The token is a GitHub PAT with the write:packages scope." - exit 1 - fi - - mkdir -p /kaniko/.docker - - | - echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"$(printf '%s:%s' "$GITLAB_CI_GHCR_USER" "$GITLAB_CI_GHCR_TOKEN" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - > - /kaniko/executor - --context "$CI_PROJECT_DIR" - --dockerfile "$CI_PROJECT_DIR/Dockerfile" + docker build --build-arg GIT_COMMIT=$CI_COMMIT_SHA - --destination "$IMAGE:sha-$CI_COMMIT_SHORT_SHA" - --destination "$IMAGE:dev-latest" + -t "$IMAGE:sha-$CI_COMMIT_SHORT_SHA" + -t "$IMAGE:dev-latest" + . + - docker push "$IMAGE:sha-$CI_COMMIT_SHORT_SHA" + - docker push "$IMAGE:dev-latest" # --------------------------------------------------------------------------- # bump-dev — no cluster access. Commits the just-built tag into the overlay @@ -176,7 +101,7 @@ bump-dev: apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev + - name: vncmail-plus newName: $IMAGE newTag: $TAG EOF @@ -215,7 +140,7 @@ bump-prod: apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev + - name: vncmail-plus newName: $IMAGE newTag: $TAG EOF diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index b4325fa1..a11f4b7e 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -59,10 +59,9 @@ is CI's job to decide — it's an explicit, human-triggered event. | 7 | Ingress `vncmail-plus` | `base/ingress.yaml` (+ overlay patches for prod) | TLS host | **Image:** CI builds and pushes to `registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus` -(tag `sha-` per deploy, moving pointers `dev-latest`/`prod-latest`). The -`ghcr.io/brvncde-dotcom/vncmail-plus-dev` image referenced in `base/deployment.yaml` -is a legacy default only — CI overrides it per-deploy via `kubectl set image`, -so what's committed there never needs to track what's actually running. +(tag `sha-` per deploy, moving pointer `dev-latest`). The generic `vncmail-plus` +image name in `base/deployment.yaml` is a placeholder — kustomize's image-tag +Component replaces it with the real registry path on every deploy. --- @@ -98,15 +97,15 @@ pick up the fix): ```bash cd deploy/k8s/overlays/dev # or overlays/prod, once real -# a) Image-pull secret — the registry package is private. -kubectl create secret docker-registry ghcr-pull \ +# a) Image-pull secret — the GitLab registry requires authentication. +# Use a project deploy token with `read_registry` scope, or the CI job +# token (short-lived — better for CI, not for long-running clusters). +kubectl create secret docker-registry gitlab-registry \ --namespace vncmail \ - --docker-server=ghcr.io \ - --docker-username=brvncde-dotcom \ - --docker-password='' \ - --docker-email=br@vnc.biz -# Once CI has cut over to registry.gitlab.vnc.biz, this becomes a -# docker-registry secret for that registry instead — see VNCMAIL-SETUP.md. + --docker-server=registry.gitlab.vnc.biz \ + --docker-username= \ + --docker-password='' \ + --docker-email=ci@vnc.biz # b) App config secret — copy the template, set a real SESSION_SECRET, apply. cp secret.example.yaml secret.yaml @@ -117,8 +116,8 @@ kubectl apply -f secret.yaml kubectl apply -k . ``` -> Alternative to (a): make the registry package public, then delete the -> `imagePullSecrets:` block from `base/deployment.yaml`. +> Alternative to (a): make the GitLab container registry public for this +> project, then delete the `imagePullSecrets:` block from `base/deployment.yaml`. After this one-time setup, routine deploys to `dev` happen automatically via CI on every push — see "Routine deploys go through CI now" above. This @@ -170,7 +169,7 @@ ArgoCD re-sync. | Symptom | Cause / fix | |---------|-------------| -| Pod `ImagePullBackOff` | `ghcr-pull` secret missing/expired, or package still private. Recreate the secret (§3a) or make the package public. | +| Pod `ImagePullBackOff` | `gitlab-registry` secret missing/expired, or token lacks `read_registry`. Recreate the secret (§3a) or make the registry public. | | Pod `CrashLoopBackOff`, logs show `EACCES`/permission on `/app/data` | Volume not writable by uid 1001. `securityContext.fsGroup: 1001` is set in `base/deployment.yaml` — keep it; some storage drivers also need it on the PVC. | | PVC stuck `Pending` | Wrong `storageClassName` in `base/pvc.yaml`. Set it to one from `kubectl get sc`. | | Ingress has no address / no cert | Wrong `ingressClassName` or cert issuer. Match bulwark's (§2). Check `kubectl -n vncmail describe ingress vncmail-plus`. | diff --git a/deploy/k8s/base/deployment.yaml b/deploy/k8s/base/deployment.yaml index b0ad23c0..cc41423c 100644 --- a/deploy/k8s/base/deployment.yaml +++ b/deploy/k8s/base/deployment.yaml @@ -23,20 +23,18 @@ spec: fsGroup: 1001 runAsUser: 1001 runAsGroup: 1001 - # Confirmed 2026-08-05: ghcr.io/brvncde-dotcom/vncmail-plus-dev IS public - # (anonymous token pull succeeded) — no imagePullSecrets needed. This is - # deploy/k8s/README.md's own documented alternative to creating a - # ghcr-pull secret. Removed rather than left referencing a - # not-yet-created secret, which would otherwise block every pod from - # starting regardless of the image being public (kubelet fails to - # resolve a missing imagePullSecrets entry before it ever gets to - # deciding whether auth was actually required). + # The GitLab container registry is private by default. Nodes need a + # docker-registry secret named `gitlab-registry` in the target namespace. + # Create it once per environment during first-time setup + # (see deploy/k8s/README.md §3a). + imagePullSecrets: + - name: gitlab-registry containers: - name: vncmail-plus - # Default/legacy value — CI overrides the image per-deploy via - # `kustomize edit set image`, so what's committed here never goes - # stale. For a one-off manual apply, pin a digest instead of :latest. - image: ghcr.io/brvncde-dotcom/vncmail-plus-dev:latest + # Generic placeholder — the real image name + tag are injected by the + # image-tag kustomize Component on every deploy (see + # overlays/*/image-tag/kustomization.yaml, rewritten by CI). + image: vncmail-plus:latest imagePullPolicy: Always ports: - containerPort: 3000 diff --git a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml index 0f6cd8a1..1fdb4a71 100644 --- a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml +++ b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml @@ -6,6 +6,6 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev - newName: ghcr.io/brvncde-dotcom/vncmail-plus-dev + - name: vncmail-plus + newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus newTag: sha-147660a diff --git a/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml b/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml index 05011284..2203d87c 100644 --- a/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml +++ b/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml @@ -4,19 +4,6 @@ # is pure waste - the content behind that tag can never change, so re-pulling # it on every pod start only adds a registry round-trip and a hard dependency # on the registry being reachable at scheduling time. -# -# It is also load-bearing right now: until CI can actually push (GitLab's -# registry vhost serves Rails, not the registry - see .gitlab-ci.yml's -# "Registry history" note), sha- tagged images are side-loaded straight into -# each node's containerd: -# -# docker save --platform linux/amd64 -o vncmail.tar : -# scp vncmail.tar dev-k8s-N:/tmp/ && ssh dev-k8s-N \ -# 'microk8s ctr images import /tmp/vncmail.tar' -# -# imported to ALL of dev-k8s-1/2/3 so the pod can schedule anywhere. With -# Always, kubelet would ignore that local image and fail on a registry pull -# for a tag the registry has never seen. apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/k8s/overlays/prod/image-tag/kustomization.yaml b/deploy/k8s/overlays/prod/image-tag/kustomization.yaml index 7619b621..d8191fe5 100644 --- a/deploy/k8s/overlays/prod/image-tag/kustomization.yaml +++ b/deploy/k8s/overlays/prod/image-tag/kustomization.yaml @@ -7,6 +7,6 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev + - name: vncmail-plus newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus newTag: not-yet-promoted