From 2e8bb9983a1836d1afa4ac32292a9edbbedf3d04 Mon Sep 17 00:00:00 2001 From: Bernd Rodler Date: Wed, 5 Aug 2026 19:40:49 +0200 Subject: [PATCH] fix(ci): back to GHCR - GitLab's registry vhost serves Rails, not the registry Diagnosed definitively rather than by log-guessing this time: $ curl -i https://registry.gitlab.vnc.biz/v2/ www-authenticate: Bearer realm="http://gitlab.vnc.biz/jwt/auth", service="dependency_proxy" x-runtime: 0.020470 x-gitlab-meta: {"correlation_id":...} x-runtime/x-gitlab-meta are Rails headers and the service is "dependency_proxy" - nginx routes that hostname to the GitLab Rails app, which treats /v2/ as the Docker Hub pull-through cache, not as this project's container registry. The registry service was never wired behind the vhost, which is why an unscoped docker login succeeded while kaniko's scoped :push request got 403 (the dependency proxy has no push concept). Fixing that is server-side nginx/omnibus work. Keeping kaniko (it solved the real dind-needs-privileged problem) and pointing it at GHCR, plus an upfront credential check so a missing variable fails in seconds instead of after a full Next.js build. --- .gitlab-ci.yml | 73 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ae76c51..05a84df6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,22 +34,42 @@ # known amd64 microk8s clusters, not public multi-arch distribution (that's # what the GHCR release workflows are for, untouched by this file). # -# Registry history (so nobody re-litigates this from scratch): GitLab's own -# Container Registry was the first choice, hit a dead end (registry_external_url -# alone didn't populate $CI_REGISTRY — see git history of this file around -# 2026-08-05 for the GHCR detour that followed), then got fixed server-side -# (gitlab_rails['registry_enabled'] confirmed set — the project's left sidebar -# now shows "Container Registry" under Deploy). Back on GitLab's native -# registry since then: it needs zero extra credentials (CI_REGISTRY_* are -# predefined GitLab CI variables, always present, scoped to this project -# only), which is strictly better than holding a GitHub PAT in GitLab CI/CD -# variables just to push images. +# Registry history (so nobody re-litigates this from scratch). GitLab's own +# Container Registry was tried twice and does not work on this server: +# +# Round 1: registry_external_url was unset, so $CI_REGISTRY was empty and +# docker login silently fell through to Docker Hub. +# Round 2: after the server-side config, the project's sidebar DID show +# "Container Registry" and docker login DID succeed - but the push +# failed 403. Diagnosed 2026-08-05 by curling the vhost directly: +# +# $ curl -i https://registry.gitlab.vnc.biz/v2/ +# www-authenticate: Bearer realm="http://gitlab.vnc.biz/jwt/auth", +# service="dependency_proxy" +# x-runtime: 0.020470 +# x-gitlab-meta: {"correlation_id":...} +# +# x-runtime/x-gitlab-meta are RAILS headers, and the service is +# "dependency_proxy" - so nginx routes that hostname to the GitLab +# Rails app, which reads /v2/ as the dependency proxy (a Docker Hub +# pull-through cache), NOT to the registry container. The registry +# service was never actually wired behind that vhost. That is why +# login worked (Rails issues an unscoped dependency-proxy token) +# while a scoped :push request 403'd - the dependency proxy has no +# push concept at all. +# +# Fixing that is an nginx/omnibus change on the GitLab server (registry +# service must actually listen behind registry.gitlab.vnc.biz), not something +# any .gitlab-ci.yml can reach. Until someone does that, GHCR it is - the +# same image the sandbox already pulls, and confirmed public so the cluster +# needs no imagePullSecrets (see deploy/k8s/base/deployment.yaml). # # Prerequisite this file assumes (documented in VNCMAIL-SETUP.md, not # something this file can set up itself): -# - GitLab Container Registry enabled for this project (confirmed 2026-08-05 -# — "Container Registry" appears in the project's left sidebar under -# Deploy / Packages and registries). +# - A GitHub PAT with `write:packages` for ghcr.io/brvncde-dotcom as +# $GITLAB_CI_GHCR_TOKEN, plus the matching GitHub username as +# $GITLAB_CI_GHCR_USER — both masked+protected CI/CD variables. A GitHub +# credential can only come from GitHub; nothing GitLab-native substitutes. # - A GitLab Runner (any kind — no cluster access needed at all now). # - Either "allow this job token to push to this project" enabled # (Settings → CI/CD → Job token permissions), OR a project access token @@ -67,11 +87,10 @@ stages: - bump-prod variables: - # GitLab's own registry, scoped to this project. $CI_REGISTRY_IMAGE is a - # predefined GitLab CI variable (populated automatically once the Container - # Registry is enabled for the project) — no manual image name to keep in - # sync, no external credential. - IMAGE: $CI_REGISTRY_IMAGE + # GHCR, not GitLab's own registry - see the "Registry history" note in the + # header for the curl output proving why. Same image the sandbox already + # pulls today. + IMAGE: ghcr.io/brvncde-dotcom/vncmail-plus-dev GIT_STRATEGY: clone # --------------------------------------------------------------------------- @@ -117,14 +136,20 @@ build: rules: - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"' script: - # $CI_REGISTRY / $CI_REGISTRY_USER / $CI_REGISTRY_PASSWORD are predefined - # GitLab CI variables, populated automatically now that this project's - # Container Registry is enabled — same credentials the old docker-login - # step already proved work, just handed to kaniko's own config.json - # instead of a daemon's. + # Fail loudly and immediately if the GHCR credentials aren't configured, + # rather than letting kaniko get all the way through a full Next.js build + # and only then 403 on push (which is exactly how the GitLab-registry + # attempt burned several pipeline runs). + - | + if [ -z "$GITLAB_CI_GHCR_TOKEN" ] || [ -z "$GITLAB_CI_GHCR_USER" ]; then + echo "ERROR: \$GITLAB_CI_GHCR_TOKEN and/or \$GITLAB_CI_GHCR_USER are not set." + echo "Add both under Settings -> CI/CD -> Variables (masked + protected)." + echo "The token is a GitHub PAT with the write:packages scope." + exit 1 + fi - mkdir -p /kaniko/.docker - | - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(printf '%s:%s' "$CI_REGISTRY_USER" "$CI_REGISTRY_PASSWORD" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"$(printf '%s:%s' "$GITLAB_CI_GHCR_USER" "$GITLAB_CI_GHCR_TOKEN" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json - > /kaniko/executor --context "$CI_PROJECT_DIR"