brvncde-dotcom
@@ -6,6 +6,10 @@ node_modules
|
||||
!.env.example
|
||||
!.env.dev.example
|
||||
scripts/
|
||||
# ...except the first-party plugin builder, which the image build runs
|
||||
# (see Dockerfile). Without this the whole scripts/ dir is absent from the
|
||||
# build context and the RUN step fails with "Cannot find module".
|
||||
!scripts/build-plugins.mjs
|
||||
TODO.md
|
||||
*.md
|
||||
!README.md
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
name: Build Electron Desktop App
|
||||
|
||||
# Phase 1 of the VNCprodbuild rollout (~/.claude/skills/VNCprodbuild/SKILL.md
|
||||
# on the machine that authored this - Phase 1 step 8). Builds the desktop
|
||||
# shell (electron/) for macOS, Windows, and Linux on every release, or
|
||||
# on-demand via workflow_dispatch for a one-off test build.
|
||||
#
|
||||
# Ships UNSIGNED. There's no Apple Developer ID or Windows code-signing cert
|
||||
# yet (VNCprodbuild Phase 1 step 9 - both are human-owned purchases, not
|
||||
# something CI can provide). CSC_IDENTITY_AUTO_DISCOVERY: "false" below stops
|
||||
# electron-builder from probing for a macOS signing identity it won't find.
|
||||
# Adding real certs later needs no rewrite here - just add CSC_LINK/
|
||||
# CSC_KEY_PASSWORD (macOS) and/or WIN_CSC_LINK/WIN_CSC_KEY_PASSWORD (Windows)
|
||||
# as repo secrets and electron-builder picks them up automatically.
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-latest, windows-latest, ubuntu-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build standalone Next.js server
|
||||
run: npm run build:standalone
|
||||
|
||||
- name: Bundle Electron main/preload
|
||||
run: npm run build:electron
|
||||
|
||||
# Only Linux runners lack a display server by default - macOS/Windows
|
||||
# GitHub-hosted runners can launch a real (if headless) GUI session
|
||||
# without one.
|
||||
- name: Install Xvfb (Linux)
|
||||
if: runner.os == 'Linux'
|
||||
run: sudo apt-get update && sudo apt-get install -y xvfb
|
||||
|
||||
# Required gate (VNCprodbuild Phase 1 step 2) before any packaging or
|
||||
# artifact-upload step below, on every OS in the matrix - a
|
||||
# platform-specific regression in electron/main.ts (path handling,
|
||||
# spawn behavior, etc.) should fail exactly the leg it breaks, not
|
||||
# slip through because only one OS was ever smoke-tested.
|
||||
- name: Run Electron smoke test (Linux, via Xvfb)
|
||||
if: runner.os == 'Linux'
|
||||
run: xvfb-run --auto-servernum npm run test:electron
|
||||
|
||||
- name: Run Electron smoke test
|
||||
if: runner.os != 'Linux'
|
||||
run: npm run test:electron
|
||||
|
||||
- name: Package
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CSC_IDENTITY_AUTO_DISCOVERY: "false"
|
||||
run: npx electron-builder --config electron-builder.config.js --publish ${{ github.event_name == 'release' && 'always' || 'never' }}
|
||||
|
||||
- name: Upload artifact (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: vncmail-plus-desktop-${{ matrix.os }}
|
||||
path: |
|
||||
dist-electron-builds/*.dmg
|
||||
dist-electron-builds/*.zip
|
||||
dist-electron-builds/*.exe
|
||||
dist-electron-builds/*.AppImage
|
||||
dist-electron-builds/*.deb
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
@@ -0,0 +1,28 @@
|
||||
name: PR Verify
|
||||
|
||||
# Required status check on `main` (Settings -> Branches). Mirrors the GitLab
|
||||
# CI `verify` stage (.gitlab-ci.yml) so both remotes gate merges the same
|
||||
# way: typecheck, lint, translations, and a real production build — no
|
||||
# registry, no cluster, nothing that can be blocked by infra that's down.
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- dev
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run typecheck
|
||||
- run: npm run lint
|
||||
- run: npm run test:translations
|
||||
- run: npm run build
|
||||
env:
|
||||
GIT_COMMIT: ${{ github.sha }}
|
||||
+22
-2
@@ -38,6 +38,14 @@ yarn-error.log*
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# electron (see electron/, scripts/build-electron.mjs, electron-builder.config.js)
|
||||
/dist-electron/
|
||||
/dist-electron-builds/
|
||||
|
||||
# playwright output
|
||||
/test-results/
|
||||
/playwright-report/
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
@@ -51,5 +59,17 @@ next-env.d.ts
|
||||
# Sibling repos
|
||||
/repos/
|
||||
|
||||
# k8s deploy secret (create from deploy/k8s/secret.example.yaml)
|
||||
/deploy/k8s/secret.yaml
|
||||
# k8s deploy secrets (create from the matching overlay's secret.example.yaml)
|
||||
/deploy/k8s/overlays/*/secret.yaml
|
||||
|
||||
# First-party plugin build output (rebuild with: npm run build:plugins).
|
||||
# vnc/plugins/build/ is the staging dir the server installs from at startup
|
||||
# (see lib/admin/bundled-plugins.ts) - built, never committed.
|
||||
vnc/plugins/build/
|
||||
vnc/plugins/*/node_modules/
|
||||
vnc/plugins/*/dist/
|
||||
vnc/plugins/smime/smime-vnc.zip
|
||||
vnc/plugins/smime/smime.zip
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
# GitLab-CI dev→prod pipeline for VNCmail+ — GitOps via ArgoCD.
|
||||
#
|
||||
# Revised after direct inspection of the real infrastructure found ArgoCD
|
||||
# already installed (idle, zero Applications) on the dev-k8s-1/2/3 cluster.
|
||||
# That's more idiomatic than a runner-executes-kubectl design, and it means
|
||||
# this pipeline needs ZERO cluster credentials — CI only ever talks to the
|
||||
# container registry and to this git repo. ArgoCD (which already has
|
||||
# whatever cluster access it needs, set up once when its Applications were
|
||||
# registered — see deploy/argocd/) is what actually applies anything.
|
||||
#
|
||||
# Design:
|
||||
# - One image name, environment lives only in the tag. No more -dev/-beta
|
||||
# name confusion from the old GitHub Actions workflow.
|
||||
# - MR into `dev`: verify only (typecheck/lint/unit test/build check). No
|
||||
# push, no deploy — this is the multi-developer merge gate.
|
||||
# - Push to `dev`: build+push an immutable `sha-<sha>` tag, then commit a
|
||||
# one-line tag-bump into overlays/dev/image-tag/kustomization.yaml
|
||||
# (`[skip ci]`, so this doesn't retrigger itself). ArgoCD's `vncmail-dev`
|
||||
# Application has automated sync — it notices the git change and applies
|
||||
# it. No approval needed, dev always deploys, and this job never touches
|
||||
# the cluster directly.
|
||||
# - Push to `main`: NEVER rebuilds. `main` only ever advances via
|
||||
# `git merge --ff-only dev`, so main's HEAD commit already has a built
|
||||
# image (the same sha- tag dev already deployed). This job just bumps
|
||||
# overlays/prod/image-tag/kustomization.yaml to point at that same tag.
|
||||
# The actual promotion gate is a HUMAN clicking Sync on the `vncmail-prod` ArgoCD
|
||||
# Application (deliberately NOT automated sync) — not a GitLab manual
|
||||
# job, since ArgoCD already provides that exact gate more directly.
|
||||
# Until prod Stalwart/hostname/secrets are real (see VNCMAIL-SETUP.md),
|
||||
# nobody should click that Sync button — but nothing here does it for
|
||||
# you either way.
|
||||
#
|
||||
# Deliberately single-platform (linux/amd64) — this pipeline serves two
|
||||
# known amd64 microk8s clusters, not public multi-arch distribution (that's
|
||||
# what the GHCR release workflows are for, untouched by this file).
|
||||
#
|
||||
# Registry history (so nobody re-litigates this from scratch). GitLab's own
|
||||
# Container Registry was tried twice and does not work on this server:
|
||||
#
|
||||
# Round 1: registry_external_url was unset, so $CI_REGISTRY was empty and
|
||||
# docker login silently fell through to Docker Hub.
|
||||
# Round 2: after the server-side config, the project's sidebar DID show
|
||||
# "Container Registry" and docker login DID succeed - but the push
|
||||
# failed 403. Diagnosed 2026-08-05 by curling the vhost directly:
|
||||
#
|
||||
# $ curl -i https://registry.gitlab.vnc.biz/v2/
|
||||
# www-authenticate: Bearer realm="http://gitlab.vnc.biz/jwt/auth",
|
||||
# service="dependency_proxy"
|
||||
# x-runtime: 0.020470
|
||||
# x-gitlab-meta: {"correlation_id":...}
|
||||
#
|
||||
# x-runtime/x-gitlab-meta are RAILS headers, and the service is
|
||||
# "dependency_proxy" - so nginx routes that hostname to the GitLab
|
||||
# Rails app, which reads /v2/ as the dependency proxy (a Docker Hub
|
||||
# pull-through cache), NOT to the registry container. The registry
|
||||
# service was never actually wired behind that vhost. That is why
|
||||
# login worked (Rails issues an unscoped dependency-proxy token)
|
||||
# while a scoped :push request 403'd - the dependency proxy has no
|
||||
# push concept at all.
|
||||
#
|
||||
# Fixing that is an nginx/omnibus change on the GitLab server (registry
|
||||
# service must actually listen behind registry.gitlab.vnc.biz), not something
|
||||
# any .gitlab-ci.yml can reach. Until someone does that, GHCR it is - the
|
||||
# same image the sandbox already pulls, and confirmed public so the cluster
|
||||
# needs no imagePullSecrets (see deploy/k8s/base/deployment.yaml).
|
||||
#
|
||||
# Prerequisite this file assumes (documented in VNCMAIL-SETUP.md, not
|
||||
# something this file can set up itself):
|
||||
# - A GitHub PAT with `write:packages` for ghcr.io/brvncde-dotcom as
|
||||
# $GITLAB_CI_GHCR_TOKEN, plus the matching GitHub username as
|
||||
# $GITLAB_CI_GHCR_USER — both masked+protected CI/CD variables. A GitHub
|
||||
# credential can only come from GitHub; nothing GitLab-native substitutes.
|
||||
# - A GitLab Runner (any kind — no cluster access needed at all now).
|
||||
# - Either "allow this job token to push to this project" enabled
|
||||
# (Settings → CI/CD → Job token permissions), OR a project access token
|
||||
# with `write_repository` scope in $GITLAB_PUSH_TOKEN. The job below
|
||||
# tries CI_JOB_TOKEN first (see the script).
|
||||
#
|
||||
# deploy/k8s/ca/ (the EJBCA internal CA) is never referenced anywhere below,
|
||||
# and neither ArgoCD Application in deploy/argocd/ points at it — that stays
|
||||
# a fully manual, human-only runbook (see deploy/k8s/ca/README.md).
|
||||
|
||||
stages:
|
||||
- verify
|
||||
- build
|
||||
- bump-dev
|
||||
- bump-prod
|
||||
|
||||
variables:
|
||||
# GHCR, not GitLab's own registry - see the "Registry history" note in the
|
||||
# header for the curl output proving why. Same image the sandbox already
|
||||
# pulls today.
|
||||
IMAGE: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
||||
GIT_STRATEGY: clone
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# verify — required check on every MR into dev. No registry, no cluster.
|
||||
# ---------------------------------------------------------------------------
|
||||
verify:
|
||||
stage: verify
|
||||
image: node:24-alpine
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
script:
|
||||
- npm ci
|
||||
- npm run typecheck
|
||||
- npm run lint
|
||||
- npm run test:translations
|
||||
- npm run build
|
||||
# test:integration is deliberately NOT here — it spins up a real Stalwart
|
||||
# fixture via docker-compose, which needs an actual Docker daemon this
|
||||
# runner's Kubernetes executor doesn't provide without privileged mode
|
||||
# (see the build job below). Candidate for a separate scheduled job on a
|
||||
# differently-configured runner, not a blocker on every MR.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build — push to dev only. Builds once; main never rebuilds (see header).
|
||||
# ---------------------------------------------------------------------------
|
||||
build:
|
||||
stage: build
|
||||
# Kaniko builds OCI images without a Docker daemon, so it needs neither a
|
||||
# dind service nor a privileged pod — GitLab's own recommended approach
|
||||
# for the Kubernetes executor specifically. The docker:27-cli + dind
|
||||
# combination that was here before this got as far as a successful
|
||||
# $CI_REGISTRY login, then failed every way it was pointed
|
||||
# (unix:///var/run/docker.sock, tcp://docker:2375, tcp://localhost:2375):
|
||||
# the dind container itself was never actually listening, which on this
|
||||
# executor means it needs `privileged: true` in the runner's own
|
||||
# config.toml — a cluster/GitLab-admin setting outside this file's
|
||||
# control. Kaniko sidesteps that requirement entirely rather than chasing
|
||||
# runner permissions further, and is also the safer default on a shared
|
||||
# cluster (no privileged containers at all).
|
||||
image:
|
||||
name: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
entrypoint: [""]
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
|
||||
script:
|
||||
# Fail loudly and immediately if the GHCR credentials aren't configured,
|
||||
# rather than letting kaniko get all the way through a full Next.js build
|
||||
# and only then 403 on push (which is exactly how the GitLab-registry
|
||||
# attempt burned several pipeline runs).
|
||||
- |
|
||||
if [ -z "$GITLAB_CI_GHCR_TOKEN" ] || [ -z "$GITLAB_CI_GHCR_USER" ]; then
|
||||
echo "ERROR: \$GITLAB_CI_GHCR_TOKEN and/or \$GITLAB_CI_GHCR_USER are not set."
|
||||
echo "Add both under Settings -> CI/CD -> Variables (masked + protected)."
|
||||
echo "The token is a GitHub PAT with the write:packages scope."
|
||||
exit 1
|
||||
fi
|
||||
- mkdir -p /kaniko/.docker
|
||||
- |
|
||||
echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"$(printf '%s:%s' "$GITLAB_CI_GHCR_USER" "$GITLAB_CI_GHCR_TOKEN" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
|
||||
- >
|
||||
/kaniko/executor
|
||||
--context "$CI_PROJECT_DIR"
|
||||
--dockerfile "$CI_PROJECT_DIR/Dockerfile"
|
||||
--build-arg GIT_COMMIT=$CI_COMMIT_SHA
|
||||
--destination "$IMAGE:sha-$CI_COMMIT_SHORT_SHA"
|
||||
--destination "$IMAGE:dev-latest"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# bump-dev — no cluster access. Commits the just-built tag into the overlay
|
||||
# ArgoCD watches; ArgoCD's automated sync does the actual apply.
|
||||
# ---------------------------------------------------------------------------
|
||||
bump-dev:
|
||||
stage: bump-dev
|
||||
image: alpine/git:2.47.0
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
|
||||
script:
|
||||
- TAG="sha-$CI_COMMIT_SHORT_SHA"
|
||||
- |
|
||||
cat > deploy/k8s/overlays/dev/image-tag/kustomization.yaml <<EOF
|
||||
# Owned by CI (bump-dev job in .gitlab-ci.yml) - regenerated every
|
||||
# push to dev. Do not hand-edit; edits here get overwritten.
|
||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||
kind: Component
|
||||
images:
|
||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
||||
newName: $IMAGE
|
||||
newTag: $TAG
|
||||
EOF
|
||||
- git config user.name "vncmail-ci"
|
||||
- git config user.email "ci@vnc.biz"
|
||||
- git add deploy/k8s/overlays/dev/image-tag/kustomization.yaml
|
||||
- |
|
||||
if git diff --cached --quiet; then
|
||||
echo "No change (tag already pinned) - nothing to commit"
|
||||
else
|
||||
git commit -m "chore(deploy): pin dev to $TAG [skip ci]"
|
||||
git push "https://gitlab-ci-token:${GITLAB_PUSH_TOKEN:-$CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" HEAD:dev
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# bump-prod — no cluster access, no rebuild. Points overlays/prod at the
|
||||
# exact tag already running on dev. Does NOT deploy anything: vncmail-prod's
|
||||
# ArgoCD Application has manual sync, so this only prepares what a human
|
||||
# would be syncing, it doesn't sync it.
|
||||
# ---------------------------------------------------------------------------
|
||||
bump-prod:
|
||||
stage: bump-prod
|
||||
image: alpine/git:2.47.0
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"'
|
||||
script:
|
||||
- TAG="sha-$CI_COMMIT_SHORT_SHA"
|
||||
- echo "main advanced to $CI_COMMIT_SHA (must be a dev commit, ff-only) - that image already exists as $IMAGE:$TAG"
|
||||
- |
|
||||
cat > deploy/k8s/overlays/prod/image-tag/kustomization.yaml <<EOF
|
||||
# Owned by CI (bump-prod job in .gitlab-ci.yml) - regenerated every
|
||||
# push to main. Do not hand-edit; edits here get overwritten. Bumping
|
||||
# this is NOT the same as deploying it - vncmail-prod's ArgoCD
|
||||
# Application has manual sync, see the note in the parent
|
||||
# kustomization.yaml.
|
||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||
kind: Component
|
||||
images:
|
||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
||||
newName: $IMAGE
|
||||
newTag: $TAG
|
||||
EOF
|
||||
- git config user.name "vncmail-ci"
|
||||
- git config user.email "ci@vnc.biz"
|
||||
- git add deploy/k8s/overlays/prod/image-tag/kustomization.yaml
|
||||
- |
|
||||
if git diff --cached --quiet; then
|
||||
echo "No change (tag already pinned) - nothing to commit"
|
||||
else
|
||||
git commit -m "chore(deploy): point prod overlay at $TAG (not synced - manual gate in ArgoCD) [skip ci]"
|
||||
git push "https://gitlab-ci-token:${GITLAB_PUSH_TOKEN:-$CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" HEAD:main
|
||||
fi
|
||||
+10
@@ -21,6 +21,12 @@ ENV NEXT_PUBLIC_DEFAULT_LOCALE=$NEXT_PUBLIC_DEFAULT_LOCALE
|
||||
# `git rev-parse` inside the build can't find it - CI must pass it in.
|
||||
ARG GIT_COMMIT=unknown
|
||||
ENV GIT_COMMIT=$GIT_COMMIT
|
||||
# Build the first-party plugins (vnc/plugins/*) that ship with this fork -
|
||||
# currently the audited S/MIME plugin, which the server installs into its
|
||||
# plugin registry at startup (lib/admin/bundled-plugins.ts). Each plugin has
|
||||
# its own package.json + lockfile, so this does its own npm ci.
|
||||
# Runs BEFORE next build so a broken plugin fails the image build.
|
||||
RUN node scripts/build-plugins.mjs
|
||||
RUN npx next build --webpack
|
||||
|
||||
FROM node:24-alpine AS runner
|
||||
@@ -43,6 +49,10 @@ RUN apk upgrade --no-cache && \
|
||||
COPY --from=builder /app/public ./public
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
|
||||
# Staged first-party plugin bundles. Read by path at runtime, so Next's output
|
||||
# file tracing does not carry them into .next/standalone - copy explicitly or
|
||||
# the image boots with the S/MIME policy toggle on and no plugin installed.
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/vnc/plugins/build ./vnc/plugins/build
|
||||
RUN mkdir -p /app/data/settings /app/data/admin /app/data/admin-state /app/data/telemetry && chown -R nextjs:nodejs /app/data
|
||||
USER nextjs
|
||||
EXPOSE 3000
|
||||
|
||||
+106
-13
@@ -18,8 +18,8 @@ of truth; VNCmail+ is the UI. It deploys as a **container on Kubernetes
|
||||
except `/tmp`, so Bulwark's `mkdir ./data` crashes (`ENOENT /var/task/data`).
|
||||
You cannot point its data dirs at a remote host either (they're POSIX paths,
|
||||
not URLs). Bulwark's native model is a container + persistent volumes.
|
||||
- So VNCmail+ runs as a Docker image (`ghcr.io/brvncde-dotcom/vncmail-plus-*`)
|
||||
with **4 persistent volumes**, exactly like the existing `bulwark.sandbox.vnc.de`.
|
||||
- So VNCmail+ runs as a Docker image with **4 persistent volumes**, exactly
|
||||
like the existing `bulwark.sandbox.vnc.de`.
|
||||
- JMAP calls go through **server-side `/api/*` routes** (`proxy.ts`) → server-to-
|
||||
server to Stalwart, **no browser CORS**. Config is **runtime-read**.
|
||||
|
||||
@@ -27,37 +27,130 @@ of truth; VNCmail+ is the UI. It deploys as a **container on Kubernetes
|
||||
|
||||
| Branch | Role |
|
||||
|--------|------|
|
||||
| `main` | **Production** — CI builds `…/vncmail-plus-beta`. Only updated by an explicit promote. |
|
||||
| `dev` | Integration + QA — CI builds `…/vncmail-plus-dev` on push. Default working branch. |
|
||||
| `vnc/*`| Feature branches for UI work (branch off `dev`, PR into `dev`). |
|
||||
| `main` | **Production.** Only updated by `git merge --ff-only dev`, then an explicit manual promote in CI. No prod environment exists yet — see "CI/CD" below. |
|
||||
| `dev` | Integration + QA — default working branch. Every push auto-builds and auto-deploys to the sandbox (`vncmail.sandbox.vnc.de`). |
|
||||
| `vnc/*`| Feature branches for UI work (branch off `dev`, MR into `dev` — required, gated by CI). |
|
||||
|
||||
All VNC customization lives under `vnc/` (see `vnc/VNC-CHANGES.md`).
|
||||
|
||||
## CI/CD — GitLab (canonical) + ArgoCD GitOps, Vercel-style dev→prod
|
||||
|
||||
Multiple developers work on this repo now. `.gitlab-ci.yml` on
|
||||
[gitlab.vnc.biz](https://gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus)
|
||||
(the canonical remote — GitHub `origin` is a passive mirror, not where CI or
|
||||
deploys happen) builds images and bumps a tag in git; **ArgoCD does the
|
||||
actual deploying** — already installed and idle on the `dev-k8s-1/2/3`
|
||||
cluster, discovered when standing this up. GitLab CI needs zero cluster
|
||||
credentials as a result.
|
||||
|
||||
Two real clusters, confirmed by direct inspection:
|
||||
|
||||
| Cluster | Role | Notes |
|
||||
|---|---|---|
|
||||
| `dev-k8s-1/2/3` | dev/sandbox | ~hours old when set up here. Traefik, metallb, cert-manager (`letsencrypt-staging` issuer only), **ArgoCD already running**. |
|
||||
| `node1/node2/node3` | prod (HA) | Older, rook-ceph+traefik+metallb+cert-manager, but **zero apps and zero ClusterIssuers** — genuinely a clean slate. |
|
||||
|
||||
Neither cluster had a `vncmail` namespace, `vnc-ca` namespace, or `bulwark`
|
||||
ingress — the "live sandbox at vncmail.sandbox.vnc.de" referenced earlier in
|
||||
this doc's history was aspirational (manifests + docs existed, nothing was
|
||||
ever actually applied). The ingress manifests also assumed nginx (`class:
|
||||
public`, an nginx body-size annotation) — fixed to Traefik's real
|
||||
`ingressClassName: traefik` (Traefik has no default body-size cap, so no
|
||||
replacement annotation is needed).
|
||||
|
||||
Flow:
|
||||
|
||||
1. **MR into `dev`** → `verify` stage (typecheck/lint/unit test/build).
|
||||
Required check — no push, no deploy.
|
||||
2. **Merge to `dev`** → `build` pushes one image,
|
||||
`registry.gitlab.vnc.biz/.../vncmail-plus:sha-<sha>`, then `bump-dev`
|
||||
commits that tag into `deploy/k8s/overlays/dev/image-tag/kustomization.yaml`
|
||||
(`[skip ci]`). ArgoCD's `vncmail-dev` Application picks up the git change.
|
||||
3. **Merge to `main`** (fast-forward only, see below) → `bump-prod` points
|
||||
`overlays/prod/image-tag/` at that same tag — **no rebuild**. The actual
|
||||
promotion gate is a **human clicking Sync** on the `vncmail-prod` ArgoCD
|
||||
Application, which is permanently manual-sync (never automated) — that's
|
||||
the Vercel-style "Promote to Production" button, just living in ArgoCD's
|
||||
UI instead of GitLab's.
|
||||
|
||||
### What's left to wire up (one-time, human steps)
|
||||
|
||||
1. **Add the ArgoCD deploy key to GitLab** — Project → Settings → Repository
|
||||
→ Deploy keys → add (read-only is enough):
|
||||
```
|
||||
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOURjX/Y9zfB785DyLEF1GUq4HhWujrqeXag8oxdMciq argocd@dev-k8s (vncmail-plus read-only)
|
||||
```
|
||||
Until this is added, `vncmail-dev`'s ArgoCD Application (already created,
|
||||
`kubectl -n argocd get application vncmail-dev`) shows a benign
|
||||
`ComparisonError` (SSH handshake failing) — expected, not a bug.
|
||||
2. **Let CI push tag-bumps back to this repo** — either enable "this project
|
||||
can be accessed by CI/CD job tokens from other projects" → actually
|
||||
simpler: Settings → CI/CD → Job token permissions → allow this project's
|
||||
own job token to push to itself, OR create a Project Access Token
|
||||
(`write_repository` scope) and add it as a masked CI/CD variable
|
||||
`GITLAB_PUSH_TOKEN` (the pipeline tries that first, falls back to
|
||||
`CI_JOB_TOKEN`).
|
||||
3. **One-time namespace bootstrap** (CI/ArgoCD deliberately never manage
|
||||
secret contents — see `deploy/k8s/README.md` §3):
|
||||
```bash
|
||||
# against dev-k8s (ArgoCD's CreateNamespace=true will make `vncmail` on
|
||||
# first sync, or create it yourself first — either order works)
|
||||
kubectl create secret docker-registry ghcr-pull -n vncmail ... # or make the GHCR package public
|
||||
cp deploy/k8s/overlays/dev/secret.example.yaml secret.yaml # edit SESSION_SECRET
|
||||
kubectl apply -f secret.yaml
|
||||
```
|
||||
4. **First sync** — ArgoCD UI at `https://argo.devcluster.vnc.de`
|
||||
(username `admin`, password: `kubectl -n argocd get secret
|
||||
argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d`
|
||||
— rotate it after logging in once) → `vncmail-dev` → Sync. Once that's
|
||||
clean, flip `deploy/argocd/vncmail-dev-app.yaml`'s commented-out
|
||||
`automated:` block on and re-apply, so dev auto-syncs on every push from
|
||||
then on.
|
||||
5. **Production** (later, deliberately not wired yet): decide a real
|
||||
hostname, stand up prod Stalwart, register `node1-3` as an ArgoCD-managed
|
||||
cluster, apply `deploy/argocd/vncmail-prod-app.yaml`, fill in real
|
||||
`overlays/prod` values, create a real ClusterIssuer on `node1-3` (there
|
||||
isn't one today), then click Sync once — deliberately not before.
|
||||
|
||||
Historical note: the old `-dev`/`-beta` GHCR image-name split
|
||||
(`.github/workflows/docker-publish.yml`) is retired by this — one image name
|
||||
now, environment lives only in the tag.
|
||||
|
||||
## Deploy (Kubernetes / microk8s)
|
||||
|
||||
Full runbook: **[deploy/k8s/README.md](deploy/k8s/README.md)**. In short:
|
||||
|
||||
1. CI builds the image on push to `dev`/`main` → `ghcr.io/brvncde-dotcom/vncmail-plus-dev` (`.github/workflows/docker-publish.yml`).
|
||||
2. `kubectl apply` the manifests in `deploy/k8s/` (namespace, 4 PVCs, deployment, service, ingress) + a `secret.yaml` (from `secret.example.yaml`) + a `ghcr-pull` image-pull secret.
|
||||
3. Point `vncmail.sandbox.vnc.de` DNS at the ingress; cert-manager issues TLS.
|
||||
1. CI (above) builds and pushes the image, one name/many tags, to GitLab's
|
||||
registry.
|
||||
2. `kubectl apply -k deploy/k8s/overlays/dev` (or `overlays/prod`, once real)
|
||||
— base manifests (namespace, 4 PVCs, deployment, service, ingress) live in
|
||||
`deploy/k8s/base/`, environment differences (namespace, hostname, replica
|
||||
count) are overlay patches.
|
||||
3. DNS + a `secret.yaml` (from the overlay's `secret.example.yaml`, gitignored,
|
||||
created once by hand — CI never manages secret contents) + an image-pull
|
||||
secret are the remaining manual, human, one-time steps per environment.
|
||||
|
||||
Runs alongside the existing `bulwark.sandbox.vnc.de`. Match your cluster's
|
||||
StorageClass / IngressClass / cert issuer to bulwark's (see the runbook).
|
||||
|
||||
## Deploy workflow (dev-first — ALWAYS)
|
||||
|
||||
Same flow as every other VNC/SRC repo:
|
||||
Same flow as every other VNC/SRC repo, now enforced structurally by CI rather
|
||||
than by convention:
|
||||
|
||||
1. Work on `dev` (or `vnc/*` → PR into `dev`). Push to `dev` → CI builds the `-dev` image → `kubectl -n vncmail rollout restart deploy/vncmail-plus` to pull it. QA at `vncmail.sandbox.vnc.de`.
|
||||
1. Work on `dev` (or `vnc/*` → MR into `dev`, CI-gated). Merge → auto-builds
|
||||
and auto-deploys to `vncmail.sandbox.vnc.de`. QA there.
|
||||
2. **Promote to production only on explicit go-live** — merge `dev` → `main`:
|
||||
```bash
|
||||
git log dev..main # MUST be empty — main must have nothing dev lacks (else prod would revert)
|
||||
git checkout main && git merge --ff-only dev
|
||||
git push origin main # CI builds the production image
|
||||
git push gitlab main # never GitHub — opens the manual `promote` job, does not run it
|
||||
git checkout dev
|
||||
```
|
||||
Then roll the production deployment to the new image (pin its digest — see deploy/k8s/README.md).
|
||||
Never push straight to `main`. Never let a dev→main merge silently revert prod.
|
||||
Then click `promote` in the GitLab pipeline UI (protected `production`
|
||||
environment — requires the right role) once prod actually exists (see
|
||||
"CI/CD" above). Never push straight to `main`. Never let a dev→main merge
|
||||
silently revert prod.
|
||||
|
||||
## Syncing upstream (Bulwark releases)
|
||||
|
||||
|
||||
@@ -134,12 +134,20 @@ export default function LoginPage() {
|
||||
const isMobileHandoff = Boolean(mobileRedirectUri);
|
||||
const { login, loginDemo, isLoading, error, clearError, isAuthenticated } = useAuthStore();
|
||||
const { theme, setTheme, initializeTheme } = useThemeStore(useShallow((s) => ({ theme: s.theme, setTheme: s.setTheme, initializeTheme: s.initializeTheme })));
|
||||
const { appName, jmapServerUrl: configuredServerUrl, oauthEnabled, oauthOnly, oauthClientId: globalOauthClientId, oauthIssuerUrl: globalOauthIssuerUrl, oauthScopes, rememberMeEnabled, devMode, demoMode, loginLogoLightUrl, loginLogoDarkUrl, loginCompanyName, loginImprintUrl, loginPrivacyPolicyUrl, loginWebsiteUrl, loginLogoMaxHeight, loginLogoMaxWidth, loginShowHeading, loginShowSubtitle, loginShowTotp, loginShowVersion, isLoading: configLoading, error: configError, autoSsoEnabled, embeddedMode: _embeddedMode, allowCustomJmapEndpoint, jmapServers, jmapServerAutoPickByDomain } = useConfig();
|
||||
const { appName, jmapServerUrl: configuredServerUrl, oauthEnabled, oauthOnly, oauthClientId: globalOauthClientId, oauthIssuerUrl: globalOauthIssuerUrl, oauthScopes, rememberMeEnabled, devMode, demoMode, loginLogoLightUrl, loginLogoDarkUrl, loginLogoLightUrlIsCustom, loginLogoDarkUrlIsCustom, loginCompanyName, loginImprintUrl, loginPrivacyPolicyUrl, loginWebsiteUrl, loginLogoMaxHeight, loginLogoMaxWidth, loginShowHeading, loginShowSubtitle, loginShowTotp, loginShowVersion, isLoading: configLoading, error: configError, autoSsoEnabled, embeddedMode: _embeddedMode, allowCustomJmapEndpoint, jmapServers, jmapServerAutoPickByDomain } = useConfig();
|
||||
const resolvedTheme = useThemeStore((s) => s.resolvedTheme);
|
||||
const { activeThemeId, installedThemes } = useThemeStore(useShallow((s) => ({ activeThemeId: s.activeThemeId, installedThemes: s.installedThemes })));
|
||||
// Active theme may carry its own brand logo (VNClagoon wordmark, SRC mark);
|
||||
// fall back to the globally configured login logo.
|
||||
const effLoginLogo = resolveThemeLogo(installedThemes, activeThemeId, resolvedTheme === 'dark', loginLogoLightUrl, loginLogoDarkUrl);
|
||||
// an explicitly-configured logo (Branding tab / LOGIN_LOGO_*_URL) wins
|
||||
// over that, falling back to the theme's logo only when nothing was set.
|
||||
const effLoginLogo = resolveThemeLogo(
|
||||
installedThemes,
|
||||
activeThemeId,
|
||||
resolvedTheme === 'dark',
|
||||
loginLogoLightUrl,
|
||||
loginLogoDarkUrl,
|
||||
loginLogoLightUrlIsCustom || loginLogoDarkUrlIsCustom,
|
||||
);
|
||||
|
||||
// Login logo sizing: when a max height/width is configured, drop the fixed
|
||||
// 64×64 box so the logo (e.g. a wide wordmark) can render at its true size.
|
||||
|
||||
@@ -32,6 +32,7 @@ import { usePromptDialog } from "@/hooks/use-prompt-dialog";
|
||||
import { useBrowserNavigation, type NavSnapshot } from "@/hooks/use-browser-navigation";
|
||||
import { debug } from "@/lib/debug";
|
||||
import { playNotificationSound } from "@/lib/notification-sound";
|
||||
import { isElectronShell, showElectronNotification } from "@/lib/electron-bridge";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { localizeMailboxName } from "@/lib/mailbox-label";
|
||||
import { KEYWORD_PREFIX, KEYWORD_PREFIX_LEGACY } from "@/lib/thread-utils";
|
||||
@@ -1062,7 +1063,45 @@ export default function Home() {
|
||||
debug.log('push', `[Push] Push notifications enabled for ${cleanups.length} account(s)`);
|
||||
}
|
||||
|
||||
// CATCH-UP for the desktop shell's local search index. The index's normal
|
||||
// trigger is a push StateChange (stores/email-store.ts's handleStateChange),
|
||||
// but nothing was pushed while the app was closed - and the polling
|
||||
// transport has no signal for contacts or files at all (client.ts's
|
||||
// buildStatePollingRequest covers Mailbox/Email/Calendar/CalendarEvent/
|
||||
// SieveScript only). So backfill a bounded recent window once per session,
|
||||
// after push is wired. Fire-and-forget; a no-op outside Electron.
|
||||
const catchUpTimer = setTimeout(() => {
|
||||
void (async () => {
|
||||
try {
|
||||
const { catchUpIndex } = await import('@/lib/mail-index-client');
|
||||
await catchUpIndex(
|
||||
useAccountStore.getState().getActiveAccount()?.cookieSlot,
|
||||
);
|
||||
} catch {
|
||||
/* the index is optional */
|
||||
}
|
||||
// The offline REPLICA's launch catch-up. Same reasoning as the index's,
|
||||
// plus one of its own: a `/changes` cursor cannot tell us about anything
|
||||
// that happened while the process was dead, so a cycle at launch is what
|
||||
// drains the backlog. One cycle is bounded, so a first sync of a large
|
||||
// mailbox needs several - `chainSync` runs them with a hard cap.
|
||||
//
|
||||
// Sequenced AFTER the index rather than in parallel: both write the same
|
||||
// SQLite file, and although `busy_timeout` makes concurrent writers safe,
|
||||
// there is no reason to spend the contention during first paint.
|
||||
try {
|
||||
const { chainSync } = await import('@/lib/offline-replica-client');
|
||||
await chainSync({ slot: useAccountStore.getState().getActiveAccount()?.cookieSlot });
|
||||
} catch {
|
||||
/* the replica is optional */
|
||||
}
|
||||
})();
|
||||
// Deliberately after the initial mailbox fetch settles: the catch-up is a
|
||||
// background nicety and must not compete with first paint.
|
||||
}, 4000);
|
||||
|
||||
return () => {
|
||||
clearTimeout(catchUpTimer);
|
||||
cleanups.forEach((fn) => fn());
|
||||
};
|
||||
}, [isAuthenticated, client, activeAccountId, connectedAccountsSignature, handleStateChange, setPushConnected, buildPopulatedUnifiedAccounts, refreshCrossCounts, refreshUnifiedCounts]);
|
||||
@@ -1186,13 +1225,34 @@ export default function Home() {
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [selectedEmail?.id, isScheduledView]);
|
||||
|
||||
// Handle new email notifications - play sound
|
||||
// Handle new email notifications - play sound, and (in the Electron shell)
|
||||
// fire a native OS notification. This effect is the transport-agnostic
|
||||
// "genuinely new unread mail arrived" signal - stores/email-store.ts's
|
||||
// refreshCurrentMailbox() already filters out sends/moves/drafts and only
|
||||
// sets newEmailNotification for a real new top-of-inbox message, and it
|
||||
// fires identically whether the underlying JMAP StateChange arrived over
|
||||
// the WebSocket push connection (lib/jmap/client.ts's connectWebSocket),
|
||||
// SSE, or the polling fallback - no need to duplicate this per transport.
|
||||
useEffect(() => {
|
||||
if (newEmailNotification) {
|
||||
const { emailNotificationsEnabled, emailNotificationSound, notificationSoundChoice } = useSettingsStore.getState();
|
||||
if (emailNotificationsEnabled && emailNotificationSound) {
|
||||
playNotificationSound(notificationSoundChoice);
|
||||
}
|
||||
if (emailNotificationsEnabled && isElectronShell()) {
|
||||
// Same fallback text public/sw.js's push handler already uses for
|
||||
// its (also un-translated) system notifications - a native OS
|
||||
// notification body isn't run through next-intl either way, so
|
||||
// matching that existing precedent instead of introducing new
|
||||
// translation keys for a rarely-hit fallback.
|
||||
const sender = newEmailNotification.from?.[0];
|
||||
const senderName = sender?.name || sender?.email || 'New mail';
|
||||
const body = newEmailNotification.subject || newEmailNotification.preview || '(no subject)';
|
||||
void showElectronNotification(senderName, {
|
||||
body,
|
||||
tag: `bulwark-mail:${newEmailNotification.id}`,
|
||||
});
|
||||
}
|
||||
debug.log('email', 'New email received:', newEmailNotification.subject);
|
||||
clearNewEmailNotification();
|
||||
}
|
||||
|
||||
@@ -26,6 +26,10 @@ export function PluginsTab() {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const [message, setMessage] = useState<{ type: 'success' | 'error'; text: string } | null>(null);
|
||||
// Bundle held back by the pattern scanner, awaiting an explicit admin decision.
|
||||
const [pendingScan, setPendingScan] = useState<
|
||||
{ file: File; findings: Array<{ file: string; patterns: string[] }> } | null
|
||||
>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const [policy, setPolicy] = useState<SettingsPolicy>({ ...DEFAULT_POLICY });
|
||||
const [policyDirty, setPolicyDirty] = useState(false);
|
||||
@@ -104,15 +108,17 @@ export function PluginsTab() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleUpload(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0];
|
||||
if (!file) return;
|
||||
|
||||
// Upload a bundle. The scanner may refuse it for containing patterns that are
|
||||
// expected in a vendored crypto library (openpgp.js, pkijs); in that case the
|
||||
// server returns `canOverride` and we hold the file so the admin can review
|
||||
// the findings and decide. `override` re-posts the same file with consent.
|
||||
async function uploadPlugin(file: File, override: boolean) {
|
||||
setUploading(true);
|
||||
setMessage(null);
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
if (override) formData.append('overrideWarnings', 'true');
|
||||
|
||||
try {
|
||||
const res = await apiFetch('/api/admin/plugins', {
|
||||
@@ -122,13 +128,22 @@ export function PluginsTab() {
|
||||
|
||||
const data = await res.json();
|
||||
if (res.ok) {
|
||||
const warnings = data.warnings?.length ? ` (${data.warnings.length} warning(s))` : '';
|
||||
setMessage({ type: 'success', text: `Plugin "${data.plugin.name}" installed${warnings}` });
|
||||
setPendingScan(null);
|
||||
const accepted = data.findings?.length
|
||||
? ` — ${data.findings.length} scanner finding(s) accepted and logged`
|
||||
: '';
|
||||
setMessage({ type: 'success', text: `Plugin "${data.plugin.name}" installed${accepted}` });
|
||||
await fetchPlugins();
|
||||
} else if (data.canOverride && Array.isArray(data.findings) && !override) {
|
||||
// Hold the file rather than the error: the admin needs to see WHAT
|
||||
// tripped, in WHICH file, before deciding.
|
||||
setPendingScan({ file, findings: data.findings });
|
||||
} else {
|
||||
setPendingScan(null);
|
||||
setMessage({ type: 'error', text: data.error || 'Upload failed' });
|
||||
}
|
||||
} catch {
|
||||
setPendingScan(null);
|
||||
setMessage({ type: 'error', text: 'Upload failed' });
|
||||
} finally {
|
||||
setUploading(false);
|
||||
@@ -136,6 +151,13 @@ export function PluginsTab() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleUpload(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0];
|
||||
if (!file) return;
|
||||
setPendingScan(null);
|
||||
await uploadPlugin(file, false);
|
||||
}
|
||||
|
||||
async function togglePlugin(id: string, enabled: boolean) {
|
||||
setMessage(null);
|
||||
const res = await apiFetch('/api/admin/plugins', {
|
||||
@@ -302,6 +324,51 @@ export function PluginsTab() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{pendingScan && (
|
||||
<div className="border border-warning/40 bg-warning/5 rounded-lg p-4 space-y-3">
|
||||
<div className="flex items-start gap-2">
|
||||
<AlertTriangle className="w-4 h-4 text-warning mt-0.5 flex-shrink-0" />
|
||||
<div className="space-y-1">
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
Scanner flagged <span className="font-mono">{pendingScan.file.name}</span>
|
||||
</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
These patterns can indicate malicious code, but they also appear in legitimate
|
||||
minified crypto libraries such as openpgp.js and pkijs. Review the findings before
|
||||
proceeding — installing anyway is recorded in the audit log.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ul className="space-y-1">
|
||||
{pendingScan.findings.map(f => (
|
||||
<li key={f.file} className="text-xs font-mono bg-background/60 border border-border rounded px-2 py-1">
|
||||
<span className="text-foreground">{f.file}</span>
|
||||
<span className="text-muted-foreground"> — {f.patterns.join(', ')}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
onClick={() => uploadPlugin(pendingScan.file, true)}
|
||||
disabled={uploading}
|
||||
className="inline-flex items-center gap-2 h-8 px-3 rounded-md bg-destructive text-destructive-foreground text-xs font-medium hover:bg-destructive/90 disabled:opacity-50 transition-all"
|
||||
>
|
||||
{uploading ? <Loader2 className="w-3.5 h-3.5 animate-spin" /> : <AlertTriangle className="w-3.5 h-3.5" />}
|
||||
Install anyway
|
||||
</button>
|
||||
<button
|
||||
onClick={() => { setPendingScan(null); setMessage(null); }}
|
||||
disabled={uploading}
|
||||
className="inline-flex items-center h-8 px-3 rounded-md border border-border text-xs font-medium text-foreground hover:bg-muted disabled:opacity-50 transition-all"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="border border-border rounded-lg">
|
||||
<div className="px-4 py-3 border-b border-border bg-muted/30">
|
||||
<div className="flex items-center gap-2">
|
||||
|
||||
@@ -158,15 +158,47 @@ export async function POST(request: NextRequest) {
|
||||
}
|
||||
const code = await entryFile.async('string');
|
||||
|
||||
// Security: block plugins containing dangerous JS patterns
|
||||
const warnings: string[] = [];
|
||||
for (const { pattern, label } of SUSPICIOUS_JS_PATTERNS) {
|
||||
if (pattern.test(code)) warnings.push(`Contains ${label}`);
|
||||
pattern.lastIndex = 0;
|
||||
// Security: scan for dangerous JS patterns across EVERY script in the
|
||||
// bundle, not just the entrypoint - a second .js file was previously never
|
||||
// looked at.
|
||||
//
|
||||
// The result is a reviewable finding rather than an unconditional reject.
|
||||
// Minified crypto libraries (openpgp.js, pkijs) legitimately contain these
|
||||
// patterns, so a hard block makes S/MIME and PGP plugins uninstallable.
|
||||
// This route is already admin-authenticated, so the scan is defence in
|
||||
// depth against an accidental or compromised upload, not a trust boundary:
|
||||
// an admin may proceed with `overrideWarnings`, and the override is
|
||||
// recorded in the audit log with the exact findings.
|
||||
const findings: Array<{ file: string; patterns: string[] }> = [];
|
||||
for (const [filePath, entry] of Object.entries(zip.files)) {
|
||||
if (entry.dir) continue;
|
||||
const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();
|
||||
if (ext !== '.js' && ext !== '.mjs') continue;
|
||||
const source = filePath === root + (manifest.entrypoint as string)
|
||||
? code
|
||||
: await entry.async('string');
|
||||
const hits: string[] = [];
|
||||
for (const { pattern, label } of SUSPICIOUS_JS_PATTERNS) {
|
||||
if (pattern.test(source)) hits.push(label);
|
||||
pattern.lastIndex = 0;
|
||||
}
|
||||
if (hits.length > 0) {
|
||||
findings.push({ file: filePath.slice(root.length), patterns: hits });
|
||||
}
|
||||
}
|
||||
if (warnings.length > 0) {
|
||||
|
||||
const overrideWarnings = formData.get('overrideWarnings') === 'true';
|
||||
if (findings.length > 0 && !overrideWarnings) {
|
||||
const summary = findings
|
||||
.map(f => `${f.file}: ${f.patterns.join(', ')}`)
|
||||
.join('; ');
|
||||
return NextResponse.json(
|
||||
{ error: `Plugin rejected: ${warnings.join(', ')}. These patterns are not allowed for security reasons.` },
|
||||
{
|
||||
error: `Plugin rejected: ${summary}. Review the bundle; if these are expected `
|
||||
+ `(e.g. a vendored crypto library), re-upload with "overrideWarnings" to proceed.`,
|
||||
findings,
|
||||
canOverride: true,
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
@@ -212,8 +244,20 @@ export async function POST(request: NextRequest) {
|
||||
await savePlugin(plugin, code);
|
||||
invalidateFrameOriginsCache();
|
||||
await auditLog('plugin.install', { id: plugin.id, name: plugin.name, version: plugin.version, frameOrigins: declaredFrameOrigins, httpOrigins: declaredHttpOrigins, apiPostPaths: declaredApiPostPaths }, ip);
|
||||
if (findings.length > 0) {
|
||||
// Record WHAT was waved through, not merely that an override happened -
|
||||
// otherwise the audit trail can't answer "which patterns did we accept?".
|
||||
await auditLog(
|
||||
'plugin.install.scan_override',
|
||||
{ id: plugin.id, version: plugin.version, findings },
|
||||
ip,
|
||||
);
|
||||
logger.warn('Plugin installed with scanner override', { id: plugin.id, findings });
|
||||
}
|
||||
|
||||
return NextResponse.json({ plugin });
|
||||
// Echo accepted findings back so the admin UI can confirm exactly what was
|
||||
// waved through, rather than reporting a bare success.
|
||||
return NextResponse.json(findings.length > 0 ? { plugin, findings } : { plugin });
|
||||
} catch (error) {
|
||||
logger.error('Plugin install error', { error: error instanceof Error ? error.message : 'Unknown error' });
|
||||
return NextResponse.json({ error: 'Internal server error' }, { status: 500 });
|
||||
|
||||
@@ -44,6 +44,17 @@ export async function GET(request: NextRequest) {
|
||||
return configManager.get<T>(key, fallback);
|
||||
};
|
||||
|
||||
// Whether a logo field was actually set by an operator (Branding tab,
|
||||
// an env var, or a per-domain override) rather than left at its default -
|
||||
// consumed by resolveThemeLogo() so an explicit choice here wins over the
|
||||
// active theme's own built-in logo, instead of being silently shadowed by
|
||||
// it. See lib/theme-logo.ts.
|
||||
const configSources = configManager.getAllWithSources();
|
||||
const isLogoOverridden = (key: BrandingOverrideKey): boolean =>
|
||||
typeof domainOverrides[key] === 'string' && domainOverrides[key]!.length > 0
|
||||
? true
|
||||
: configSources[key]?.source !== 'default';
|
||||
|
||||
const appName =
|
||||
branded<string>('appName', '') || process.env.NEXT_PUBLIC_APP_NAME || 'Webmail';
|
||||
const jmapServerUrl = configManager.get<string>('jmapServerUrl') || process.env.NEXT_PUBLIC_JMAP_SERVER_URL || '';
|
||||
@@ -68,8 +79,12 @@ export async function GET(request: NextRequest) {
|
||||
faviconUrl: branded<string>('faviconUrl', '/branding/Bulwark_Favicon.svg'),
|
||||
appLogoLightUrl: branded<string>('appLogoLightUrl', ''),
|
||||
appLogoDarkUrl: branded<string>('appLogoDarkUrl', ''),
|
||||
appLogoLightUrlIsCustom: isLogoOverridden('appLogoLightUrl'),
|
||||
appLogoDarkUrlIsCustom: isLogoOverridden('appLogoDarkUrl'),
|
||||
loginLogoLightUrl: branded<string>('loginLogoLightUrl', '/branding/Bulwark_Logo_Color.svg'),
|
||||
loginLogoDarkUrl: branded<string>('loginLogoDarkUrl', '/branding/Bulwark_Logo_White.svg'),
|
||||
loginLogoLightUrlIsCustom: isLogoOverridden('loginLogoLightUrl'),
|
||||
loginLogoDarkUrlIsCustom: isLogoOverridden('loginLogoDarkUrl'),
|
||||
loginCompanyName: branded<string>('loginCompanyName', ''),
|
||||
loginImprintUrl: branded<string>('loginImprintUrl', ''),
|
||||
loginPrivacyPolicyUrl: branded<string>('loginPrivacyPolicyUrl', ''),
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// GET /api/offline/mail?kind=mailboxes|list|message - the OFFLINE READ SURFACE.
|
||||
//
|
||||
// THIS ROUTE MUST NEVER MAKE A NETWORK CALL. That is the whole feature: it is
|
||||
// consulted precisely when the backend is unreachable, so a JMAP session fetch to
|
||||
// learn the account id would fail for the exact reason the route was called. The
|
||||
// account is resolved from the request's own encrypted `jmap_stalwart_ctx` cookie
|
||||
// (a local decrypt) and from the account ids the store already holds rows for.
|
||||
//
|
||||
// It is a FALLBACK, not a cache in front of the server - see `read.ts`'s header
|
||||
// for the coherence rules that depend on that, and `lib/offline-fallback-client.ts`
|
||||
// for the one place that decides a read has genuinely failed.
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import {
|
||||
readEnvelopePage, readMailboxes, readMessage,
|
||||
} from '@/lib/offline-replica/read';
|
||||
import {
|
||||
resolveIndexSession, resolveReadAccountId, withReplica,
|
||||
} from '@/lib/offline-replica/engine';
|
||||
import { gateReplicaRoute, NO_STORE, replicaErrorResponse } from '@/lib/offline-replica/route-gate';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
const MAX_LIMIT = 200;
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const gated = gateReplicaRoute();
|
||||
if (gated) return gated;
|
||||
|
||||
const params = request.nextUrl.searchParams;
|
||||
const kind = params.get('kind') ?? 'mailboxes';
|
||||
if (kind !== 'mailboxes' && kind !== 'list' && kind !== 'message') {
|
||||
return NextResponse.json({ error: 'kind must be mailboxes, list or message' }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
const session = await resolveIndexSession(request);
|
||||
const payload = await withReplica(session.accountId, (store) => {
|
||||
const jmapAccountId = resolveReadAccountId(store, params.get('jmapAccountId'));
|
||||
if (!jmapAccountId) {
|
||||
// Nothing synced yet for this account. Not an error - the caller falls
|
||||
// back to whatever it would have shown without a replica.
|
||||
return { empty: true as const };
|
||||
}
|
||||
|
||||
if (kind === 'mailboxes') {
|
||||
return { empty: false as const, jmapAccountId, mailboxes: readMailboxes(store, jmapAccountId) };
|
||||
}
|
||||
|
||||
if (kind === 'list') {
|
||||
const rawLimit = Number(params.get('limit') ?? '50');
|
||||
const limit = Number.isFinite(rawLimit)
|
||||
? Math.min(Math.max(Math.trunc(rawLimit), 1), MAX_LIMIT)
|
||||
: 50;
|
||||
const rawOffset = Number(params.get('offset') ?? '0');
|
||||
const offset = Number.isFinite(rawOffset) ? Math.max(Math.trunc(rawOffset), 0) : 0;
|
||||
// An absent mailboxId means "everything", which is what the unified views
|
||||
// ask for; an empty string is a caller bug and must not silently widen.
|
||||
const mailboxParam = params.get('mailboxId');
|
||||
const mailboxId = mailboxParam === null ? null : mailboxParam;
|
||||
if (mailboxId === '') {
|
||||
return { empty: true as const };
|
||||
}
|
||||
const page = readEnvelopePage(store, jmapAccountId, mailboxId, limit, offset);
|
||||
return { empty: false as const, jmapAccountId, ...page };
|
||||
}
|
||||
|
||||
const id = params.get('id');
|
||||
if (!id || id.length > 256) return { empty: true as const };
|
||||
const message = readMessage(store, jmapAccountId, id);
|
||||
if (!message) return { empty: false as const, jmapAccountId, email: null, hasBody: false };
|
||||
return {
|
||||
empty: false as const,
|
||||
jmapAccountId,
|
||||
email: message.email,
|
||||
hasBody: message.hasBody,
|
||||
};
|
||||
});
|
||||
|
||||
if (payload.empty) {
|
||||
return NextResponse.json({ ok: true, available: false }, { headers: NO_STORE });
|
||||
}
|
||||
return NextResponse.json({ ok: true, available: true, ...payload }, { headers: NO_STORE });
|
||||
} catch (error) {
|
||||
return replicaErrorResponse(error, 'offline read');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
// POST /api/offline/reindex - write mail/calendar/contacts/files into the
|
||||
// encrypted local search index for the calling session's account.
|
||||
//
|
||||
// The PRIMARY caller is the renderer's live JMAP push handler: when a
|
||||
// StateChange arrives it posts the ids that changed, so indexing is reactive to
|
||||
// each delivery rather than periodic. `{ catchUp: true }` (no ids) is the
|
||||
// fallback used at app launch to backfill whatever changed while the app was
|
||||
// closed.
|
||||
//
|
||||
// GATED: returns 404 unless VNCMAIL_DESKTOP_STORE_DIR is set, which only
|
||||
// electron/main.ts does. The same standalone server artifact runs in the
|
||||
// multi-tenant production Docker image, where this feature must not exist at
|
||||
// all - 404 rather than 403 so nothing learns the route is there.
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { logger } from '@/lib/logger';
|
||||
import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
||||
import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key';
|
||||
import { getStoreDir } from '@/lib/mail-index/paths';
|
||||
import {
|
||||
IndexSessionError, MAX_IDS_PER_CALL, resolveIndexSession, runIndex,
|
||||
type IndexRequest,
|
||||
} from '@/lib/mail-index/reindex';
|
||||
import { CONTENT_TYPES, isContentType, type ContentType } from '@/lib/mail-index/store';
|
||||
import { JmapIndexError } from '@/lib/mail-index/jmap';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
function parseIdMap(raw: unknown): Partial<Record<ContentType, string[]>> | undefined {
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined;
|
||||
const out: Partial<Record<ContentType, string[]>> = {};
|
||||
for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
|
||||
if (!isContentType(key) || !Array.isArray(value)) continue;
|
||||
const ids = value
|
||||
.filter((v): v is string => typeof v === 'string' && v.length > 0 && v.length <= 256)
|
||||
.slice(0, MAX_IDS_PER_CALL);
|
||||
if (ids.length > 0) out[key] = ids;
|
||||
}
|
||||
return Object.keys(out).length > 0 ? out : undefined;
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
if (!getStoreDir()) {
|
||||
return new NextResponse(null, { status: 404 });
|
||||
}
|
||||
if (!hasKeyChannel()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'The local index has no key channel in this process.', code: 'no-key-channel' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
if (!isSqlcipherAvailable()) {
|
||||
// The native binding is an optionalDependency, so "not installed" is a
|
||||
// normal state on platforms without a prebuild - not an error to log loudly.
|
||||
return NextResponse.json(
|
||||
{ error: 'Encrypted local index is unavailable on this platform.', code: 'no-binding' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown> = {};
|
||||
try {
|
||||
const text = await request.text();
|
||||
if (text.trim()) body = JSON.parse(text) as Record<string, unknown>;
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Malformed JSON body' }, { status: 400 });
|
||||
}
|
||||
|
||||
const rawTypes = Array.isArray(body.types) ? body.types.filter(isContentType) : [];
|
||||
const req: IndexRequest = {
|
||||
types: rawTypes.length > 0 ? rawTypes : undefined,
|
||||
ids: parseIdMap(body.ids),
|
||||
removed: parseIdMap(body.removed),
|
||||
// Pruning is a catch-up concern; a single-delivery call shouldn't scan.
|
||||
prune: body.catchUp === true,
|
||||
};
|
||||
|
||||
try {
|
||||
const session = await resolveIndexSession(request);
|
||||
const result = await runIndex(session, req);
|
||||
return NextResponse.json(
|
||||
{
|
||||
ok: true,
|
||||
written: result.written,
|
||||
skipped: result.skipped,
|
||||
errors: result.errors,
|
||||
durationMs: result.durationMs,
|
||||
types: CONTENT_TYPES,
|
||||
},
|
||||
{ headers: { 'Cache-Control': 'no-store' } },
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof IndexSessionError) {
|
||||
return NextResponse.json({ error: error.message }, { status: error.status });
|
||||
}
|
||||
if (error instanceof JmapIndexError) {
|
||||
return NextResponse.json({ error: error.message }, { status: error.status });
|
||||
}
|
||||
if (error instanceof IndexKeyError) {
|
||||
// no-secure-storage is the Linux-without-a-keyring refusal: a real,
|
||||
// expected outcome with a user-facing explanation, not a server fault.
|
||||
const status = error.code === 'no-secure-storage' ? 503 : 500;
|
||||
return NextResponse.json({ error: error.message, code: error.code }, { status });
|
||||
}
|
||||
logger.error('mail-index reindex failed', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return NextResponse.json({ error: 'Reindex failed' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// GET /api/offline/search?q=...&types=mail,calendar&limit=20
|
||||
//
|
||||
// THE RETRIEVAL SURFACE. This is what an AI/RAG feature calls to gather
|
||||
// relevant context from the user's own mail, calendar, contacts and files
|
||||
// before prompting a model - hence the `snippet` on every hit and the
|
||||
// `contextBlock` convenience field, which is the same information already
|
||||
// flattened into text a prompt can carry directly.
|
||||
//
|
||||
// Read-only: it never touches the network and never writes. Gated identically
|
||||
// to the reindex route.
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { logger } from '@/lib/logger';
|
||||
import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
||||
import { hasKeyChannel, IndexKeyError, withIndexKey } from '@/lib/mail-index/key';
|
||||
import { getStoreDir } from '@/lib/mail-index/paths';
|
||||
import { IndexSessionError, resolveIndexSession } from '@/lib/mail-index/reindex';
|
||||
import {
|
||||
isContentType, MailIndex, MailIndexUnavailableError, type ContentType, type SearchHit,
|
||||
} from '@/lib/mail-index/store';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
/**
|
||||
* One hit as a plain text block, ready to be concatenated into a prompt.
|
||||
* Kept server-side so every caller (a chat feature, a future agent, a test)
|
||||
* formats context the same way rather than each inventing its own.
|
||||
*/
|
||||
function toContextBlock(hit: SearchHit): string {
|
||||
const label: Record<ContentType, string> = {
|
||||
mail: 'EMAIL', calendar: 'CALENDAR EVENT', contact: 'CONTACT', file: 'FILE',
|
||||
};
|
||||
const lines = [`[${label[hit.contentType]}] ${hit.title}`];
|
||||
if (hit.occurredAt) lines.push(`Date: ${hit.occurredAt}`);
|
||||
if (hit.people) lines.push(`People: ${hit.people}`);
|
||||
const path = hit.metadata?.path;
|
||||
if (typeof path === 'string' && path) lines.push(`Path: ${path}`);
|
||||
if (hit.snippet) lines.push(`Excerpt: ${hit.snippet}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
if (!getStoreDir()) {
|
||||
return new NextResponse(null, { status: 404 });
|
||||
}
|
||||
if (!hasKeyChannel() || !isSqlcipherAvailable()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Encrypted local index is unavailable in this process.', code: 'unavailable' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const params = request.nextUrl.searchParams;
|
||||
const query = (params.get('q') ?? '').trim();
|
||||
const wantStats = params.get('stats') === 'true';
|
||||
|
||||
if (!query && !wantStats) {
|
||||
return NextResponse.json({ error: 'Missing q parameter' }, { status: 400 });
|
||||
}
|
||||
if (query.length > 512) {
|
||||
return NextResponse.json({ error: 'Query too long' }, { status: 400 });
|
||||
}
|
||||
|
||||
const types = (params.get('types') ?? '')
|
||||
.split(',')
|
||||
.map((t) => t.trim())
|
||||
.filter(isContentType);
|
||||
|
||||
const limitRaw = Number(params.get('limit') ?? '20');
|
||||
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(Math.trunc(limitRaw), 1), 100) : 20;
|
||||
|
||||
try {
|
||||
const session = await resolveIndexSession(request);
|
||||
const storeDir = getStoreDir();
|
||||
if (!storeDir) return new NextResponse(null, { status: 404 });
|
||||
|
||||
const payload = await withIndexKey(session.accountId, (key) => {
|
||||
const index = MailIndex.open({ storeDir, accountId: session.accountId, key });
|
||||
try {
|
||||
const stats = index.stats();
|
||||
if (!query) return { hits: [] as SearchHit[], stats };
|
||||
return { hits: index.search({ query, types, limit }), stats: wantStats ? stats : undefined };
|
||||
} finally {
|
||||
index.close();
|
||||
}
|
||||
});
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
ok: true,
|
||||
query,
|
||||
types: types.length > 0 ? types : 'all',
|
||||
count: payload.hits.length,
|
||||
hits: payload.hits,
|
||||
// Everything a prompt needs, pre-joined in rank order.
|
||||
contextBlock: payload.hits.map(toContextBlock).join('\n\n---\n\n'),
|
||||
...(payload.stats ? { stats: payload.stats } : {}),
|
||||
},
|
||||
{ headers: { 'Cache-Control': 'no-store' } },
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof IndexSessionError) {
|
||||
return NextResponse.json({ error: error.message }, { status: error.status });
|
||||
}
|
||||
if (error instanceof IndexKeyError) {
|
||||
const status = error.code === 'no-secure-storage' ? 503 : 500;
|
||||
return NextResponse.json({ error: error.message, code: error.code }, { status });
|
||||
}
|
||||
if (error instanceof MailIndexUnavailableError) {
|
||||
return NextResponse.json({ error: error.message, code: 'unavailable' }, { status: 503 });
|
||||
}
|
||||
logger.error('mail-index search failed', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return NextResponse.json({ error: 'Search failed' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// GET /api/offline/status - size, freshness and retention policy, for Settings.
|
||||
// PUT /api/offline/status - update the retention policy.
|
||||
// DELETE /api/offline/status - purge the replica.
|
||||
//
|
||||
// The POLICY LIVES IN THE ENCRYPTED STORE, not in renderer localStorage. The
|
||||
// design review's H1 was that a server-side engine cannot read a renderer-only
|
||||
// setting; keeping the policy server-side means the retention pass always has the
|
||||
// value it needs, while the DECISION TO SYNC AT ALL stays with the renderer, so
|
||||
// nothing is ever materialised for an account that never opted in.
|
||||
//
|
||||
// Like every read here, GET makes no network call: an offline user must still be
|
||||
// able to see what they have and free the space.
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { clampPolicy, POLICY_LIMITS, type RetentionPolicy } from '@/lib/offline-replica/store';
|
||||
import { resolveIndexSession, resolveReadAccountId, withReplica } from '@/lib/offline-replica/engine';
|
||||
import { gateReplicaRoute, NO_STORE, replicaErrorResponse } from '@/lib/offline-replica/route-gate';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const gated = gateReplicaRoute();
|
||||
if (gated) return gated;
|
||||
try {
|
||||
const session = await resolveIndexSession(request);
|
||||
const payload = await withReplica(session.accountId, (store) => {
|
||||
const jmapAccountId = resolveReadAccountId(store, null);
|
||||
const policy = store.getPolicy();
|
||||
const flags = store.getFlags(Date.now());
|
||||
if (!jmapAccountId) {
|
||||
return {
|
||||
policy,
|
||||
limits: POLICY_LIMITS,
|
||||
synced: false,
|
||||
stats: null,
|
||||
coveragePhase: 'never-run',
|
||||
resyncRequired: flags.resyncRequired,
|
||||
lastCycleAt: flags.lastCycleAt ?? null,
|
||||
lastCycleOk: flags.lastCycleOk ?? null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
policy,
|
||||
limits: POLICY_LIMITS,
|
||||
synced: true,
|
||||
stats: store.stats(jmapAccountId),
|
||||
coveragePhase: store.getCoverage(jmapAccountId)?.phase ?? 'never-run',
|
||||
coveredFrom: store.getCoverage(jmapAccountId)?.coveredFrom ?? null,
|
||||
resyncRequired: flags.resyncRequired,
|
||||
lastCycleAt: flags.lastCycleAt ?? null,
|
||||
lastCycleOk: flags.lastCycleOk ?? null,
|
||||
lastCycleError: flags.lastCycleError ?? null,
|
||||
};
|
||||
});
|
||||
return NextResponse.json({ ok: true, ...payload }, { headers: NO_STORE });
|
||||
} catch (error) {
|
||||
return replicaErrorResponse(error, 'offline status');
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest) {
|
||||
const gated = gateReplicaRoute();
|
||||
if (gated) return gated;
|
||||
|
||||
let body: Record<string, unknown> = {};
|
||||
try {
|
||||
const text = await request.text();
|
||||
if (text.trim()) body = JSON.parse(text) as Record<string, unknown>;
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Malformed JSON body' }, { status: 400 });
|
||||
}
|
||||
|
||||
const policy = clampPolicy(body as Partial<RetentionPolicy>);
|
||||
try {
|
||||
const session = await resolveIndexSession(request);
|
||||
await withReplica(session.accountId, (store) => {
|
||||
store.transaction(() => { store.setPolicy(policy); });
|
||||
});
|
||||
// The cycle applies it: a widen re-enters coverage scanning, a narrow evicts,
|
||||
// and the clock guard is told this was INTENT rather than a glitch by the
|
||||
// `lastEnvelopeDays` it compares against.
|
||||
return NextResponse.json({ ok: true, policy }, { headers: NO_STORE });
|
||||
} catch (error) {
|
||||
return replicaErrorResponse(error, 'offline policy update');
|
||||
}
|
||||
}
|
||||
|
||||
export async function DELETE(request: NextRequest) {
|
||||
const gated = gateReplicaRoute();
|
||||
if (gated) return gated;
|
||||
try {
|
||||
const session = await resolveIndexSession(request);
|
||||
await withReplica(session.accountId, (store) => {
|
||||
// ALL OF IT, cursors included. A record wipe that leaves cursors behind is
|
||||
// the one state no amount of syncing repairs: `/changes` structurally cannot
|
||||
// re-deliver mail that already existed when the cursor was captured, so the
|
||||
// next cycle would advance a live cursor over an empty store forever.
|
||||
store.transaction(() => { store.purgeAll(); });
|
||||
});
|
||||
return NextResponse.json({ ok: true, purged: true }, { headers: NO_STORE });
|
||||
} catch (error) {
|
||||
return replicaErrorResponse(error, 'offline purge');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// POST /api/offline/sync - run ONE bounded delta-sync cycle for the calling
|
||||
// session's account.
|
||||
//
|
||||
// The renderer drives this: once at launch (catch-up for whatever changed while
|
||||
// the app was closed, for which no push event was ever delivered) and on each
|
||||
// JMAP `StateChange` from the live push connection. There is no background worker
|
||||
// and no resident credential - see `lib/offline-replica/sync.ts`'s header for why
|
||||
// that architecture choice keeps most of the original design review's critical
|
||||
// findings out of scope entirely.
|
||||
//
|
||||
// A cycle is BOUNDED. `unfinishedWork: true` means "call again", and the renderer
|
||||
// chains with a cap; it never means an error.
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { clampPolicy, type RetentionPolicy } from '@/lib/offline-replica/store';
|
||||
import { resolveIndexSession, syncAccount } from '@/lib/offline-replica/engine';
|
||||
import { gateReplicaRoute, NO_STORE, replicaErrorResponse } from '@/lib/offline-replica/route-gate';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const gated = gateReplicaRoute();
|
||||
if (gated) return gated;
|
||||
|
||||
let body: Record<string, unknown> = {};
|
||||
try {
|
||||
const text = await request.text();
|
||||
if (text.trim()) body = JSON.parse(text) as Record<string, unknown>;
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Malformed JSON body' }, { status: 400 });
|
||||
}
|
||||
|
||||
const rawPolicy = body.policy;
|
||||
const policy: RetentionPolicy | undefined =
|
||||
rawPolicy && typeof rawPolicy === 'object' && !Array.isArray(rawPolicy)
|
||||
? clampPolicy(rawPolicy as Partial<RetentionPolicy>)
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
const session = await resolveIndexSession(request);
|
||||
const report = await syncAccount(session, {
|
||||
policy,
|
||||
forceResync: body.forceResync === true,
|
||||
});
|
||||
return NextResponse.json({ ok: report.ok, report }, { headers: NO_STORE });
|
||||
} catch (error) {
|
||||
return replicaErrorResponse(error, 'sync');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* S/MIME certificate enrolment (`C-08`, server half).
|
||||
*
|
||||
* The plugin generates a keypair in the browser and sends only a CSR here. The
|
||||
* private key never leaves the device — this route never sees it and has no way
|
||||
* to ask for it.
|
||||
*
|
||||
* What this route exists to decide: **which addresses the issued certificate is
|
||||
* allowed to assert.** That question cannot be answered in the browser, and it
|
||||
* must not be answered by the CSR — a CSR is a self-assertion, and honouring its
|
||||
* `subjectAltName` would let anyone mint a certificate for any address, which is
|
||||
* indistinguishable from having no CA at all.
|
||||
*/
|
||||
import { NextResponse } from 'next/server';
|
||||
import { readStalwartAuthContext } from '@/lib/stalwart/auth-context';
|
||||
import { fetchJmapSession, postJmap, rebaseApiUrl } from '@/lib/stalwart/jmap-api';
|
||||
import { CaError, getCaProvider } from '@/lib/smime-ca';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
const MAX_CSR_BYTES = 8 * 1024;
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const provider = getCaProvider();
|
||||
if (!provider) {
|
||||
return NextResponse.json(
|
||||
{ error: 'S/MIME enrolment is not configured on this server' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: { csrPem?: unknown; slot?: unknown };
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'invalid JSON body' }, { status: 400 });
|
||||
}
|
||||
|
||||
const csrPem = typeof body.csrPem === 'string' ? body.csrPem.trim() : '';
|
||||
if (!csrPem) {
|
||||
return NextResponse.json({ error: 'csrPem is required' }, { status: 400 });
|
||||
}
|
||||
if (csrPem.length > MAX_CSR_BYTES) {
|
||||
return NextResponse.json({ error: 'csrPem too large' }, { status: 413 });
|
||||
}
|
||||
// Shape check only. This is not a security control — see the module comment on
|
||||
// why the CSR's contents are not trusted regardless of what they contain.
|
||||
if (!/^-----BEGIN (NEW )?CERTIFICATE REQUEST-----[\s\S]+-----END (NEW )?CERTIFICATE REQUEST-----$/
|
||||
.test(csrPem)) {
|
||||
return NextResponse.json({ error: 'csrPem is not a PEM PKCS#10 request' }, { status: 400 });
|
||||
}
|
||||
|
||||
const slot = Number.isInteger(body.slot) ? (body.slot as number) : 0;
|
||||
if (slot < 0 || slot > 9) {
|
||||
return NextResponse.json({ error: 'invalid slot' }, { status: 400 });
|
||||
}
|
||||
|
||||
// The auth context is an encrypted, server-minted cookie, so `username` cannot
|
||||
// be forged by the client. It still isn't sufficient on its own — see below.
|
||||
const auth = await readStalwartAuthContext(slot);
|
||||
if (!auth) {
|
||||
return NextResponse.json({ error: 'not authenticated' }, { status: 401 });
|
||||
}
|
||||
|
||||
let identity: { addresses: string[]; displayName?: string };
|
||||
try {
|
||||
identity = await resolveIdentity(auth.serverUrl, auth.authHeader);
|
||||
} catch (cause) {
|
||||
console.error('[smime-enroll] identity resolution failed:', cause);
|
||||
return NextResponse.json(
|
||||
{ error: 'could not confirm your sending addresses with the mail server' },
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
|
||||
if (identity.addresses.length === 0) {
|
||||
// An authenticated principal with no sending identity — an admin-only
|
||||
// account, or a mailbox with submission disabled. Refuse rather than falling
|
||||
// back to the cookie's username, which would issue a certificate for an
|
||||
// address the mail server will not actually let this account send from.
|
||||
return NextResponse.json(
|
||||
{ error: 'this account has no sending address, so no certificate can be issued for it' },
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const issued = await provider.enroll({
|
||||
csrPem,
|
||||
addresses: identity.addresses,
|
||||
commonName: identity.displayName || identity.addresses[0],
|
||||
});
|
||||
|
||||
// Audit before returning. A certificate that exists with no record of who
|
||||
// asked for it is the thing you most want during an incident.
|
||||
console.info(
|
||||
`[smime-enroll] issued serial=${issued.serialNumber} ca=${provider.id} `
|
||||
+ `account=${auth.username} addresses=${identity.addresses.join(',')}`,
|
||||
);
|
||||
|
||||
return NextResponse.json({
|
||||
certificatePem: issued.certificatePem,
|
||||
chainPem: issued.chainPem,
|
||||
serialNumber: issued.serialNumber,
|
||||
issuerDn: issued.issuerDn,
|
||||
notAfter: issued.notAfter,
|
||||
addresses: identity.addresses,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof CaError) {
|
||||
console.error(`[smime-enroll] CA error for ${auth.username}:`, error.message, error.cause);
|
||||
return NextResponse.json({ error: error.message }, { status: error.status });
|
||||
}
|
||||
console.error('[smime-enroll] unexpected error:', error);
|
||||
return NextResponse.json({ error: 'enrolment failed' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask Stalwart which addresses this session may send from, via `Identity/get`.
|
||||
*
|
||||
* This is deliberately not derived from the auth cookie's `username`. The right
|
||||
* authority for "may this person have a signing certificate for this address" is
|
||||
* the mail server that already decides "may this person send from this address" —
|
||||
* anything else invents a second, weaker answer to a question already settled.
|
||||
*
|
||||
* It also handles the cases the cookie cannot: an alias the account legitimately
|
||||
* sends as (which should be on the certificate) and an administrative principal
|
||||
* with no mailbox at all (which should get no certificate). The latter is not
|
||||
* hypothetical here — `admin@sandbox.vnc.de` authenticates successfully and has
|
||||
* no mail session, and trusting the cookie would have issued it a certificate.
|
||||
*/
|
||||
async function resolveIdentity(
|
||||
serverUrl: string,
|
||||
authHeader: string,
|
||||
): Promise<{ addresses: string[]; displayName?: string }> {
|
||||
const session = await fetchJmapSession(serverUrl, authHeader);
|
||||
if (!session) throw new Error('no JMAP session');
|
||||
|
||||
const accountId = session.primaryAccounts?.['urn:ietf:params:jmap:mail'];
|
||||
if (!accountId) throw new Error('no primary mail account');
|
||||
|
||||
const apiUrl = rebaseApiUrl(session, serverUrl);
|
||||
if (!apiUrl) throw new Error('session advertises no usable apiUrl');
|
||||
|
||||
const res = await postJmap(apiUrl, authHeader, JSON.stringify({
|
||||
using: ['urn:ietf:params:jmap:core', 'urn:ietf:params:jmap:submission'],
|
||||
methodCalls: [['Identity/get', { accountId }, '0']],
|
||||
}));
|
||||
if (!res.ok) throw new Error(`Identity/get returned ${res.status}`);
|
||||
|
||||
const payload = await res.json() as {
|
||||
methodResponses?: [string, { list?: { email?: string; name?: string }[] }, string][];
|
||||
};
|
||||
const first = payload.methodResponses?.[0];
|
||||
if (!first || first[0] !== 'Identity/get') {
|
||||
throw new Error('Identity/get failed');
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
const addresses: string[] = [];
|
||||
let displayName: string | undefined;
|
||||
|
||||
for (const entry of first[1]?.list ?? []) {
|
||||
const email = typeof entry.email === 'string' ? entry.email.trim().toLowerCase() : '';
|
||||
// Stalwart can report a wildcard identity (`*@domain`) for accounts allowed
|
||||
// to send as anything in a domain. That is a real capability, but it is not
|
||||
// an address and must never reach a certificate — a `rfc822Name` SAN of
|
||||
// `*@vnc.de` is either rejected by clients or, worse, honoured.
|
||||
if (!email || email.includes('*') || !email.includes('@')) continue;
|
||||
if (seen.has(email)) continue;
|
||||
seen.add(email);
|
||||
addresses.push(email);
|
||||
if (!displayName && typeof entry.name === 'string' && entry.name.trim()) {
|
||||
displayName = entry.name.trim();
|
||||
}
|
||||
}
|
||||
|
||||
return { addresses, displayName };
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 100 KiB |
@@ -189,7 +189,7 @@ export function NavigationRail({
|
||||
const t = useTranslations("sidebar");
|
||||
const pathname = usePathname();
|
||||
const router = useRouter();
|
||||
const { appLogoLightUrl, appLogoDarkUrl } = useConfig();
|
||||
const { appLogoLightUrl, appLogoDarkUrl, appLogoLightUrlIsCustom, appLogoDarkUrlIsCustom } = useConfig();
|
||||
const resolvedTheme = useThemeStore((s) => s.resolvedTheme);
|
||||
const activeThemeId = useThemeStore((s) => s.activeThemeId);
|
||||
const installedThemes = useThemeStore((s) => s.installedThemes);
|
||||
@@ -465,7 +465,7 @@ export function NavigationRail({
|
||||
)}
|
||||
>
|
||||
{(() => {
|
||||
const logoUrl = withBasePath(resolveThemeLogo(installedThemes, activeThemeId, resolvedTheme === 'dark', appLogoLightUrl, appLogoDarkUrl));
|
||||
const logoUrl = withBasePath(resolveThemeLogo(installedThemes, activeThemeId, resolvedTheme === 'dark', appLogoLightUrl, appLogoDarkUrl, appLogoLightUrlIsCustom || appLogoDarkUrlIsCustom));
|
||||
return logoUrl ? (
|
||||
<div className="flex items-center justify-center py-3 px-1">
|
||||
<img
|
||||
|
||||
@@ -13,6 +13,7 @@ import { cn } from '@/lib/utils';
|
||||
import { getPathPrefix } from '@/lib/browser-navigation';
|
||||
import { clearCachedData } from '@/lib/clear-cached-data';
|
||||
import { SpamSiegeGame } from './spam-siege-game';
|
||||
import { LocalIndexSettings } from './local-index-settings';
|
||||
|
||||
const APP_VERSION = process.env.NEXT_PUBLIC_APP_VERSION || "0.0.0";
|
||||
const GIT_COMMIT = process.env.NEXT_PUBLIC_GIT_COMMIT || "unknown";
|
||||
@@ -218,6 +219,9 @@ export function AboutDataSettings() {
|
||||
</Button>
|
||||
</SettingItem>
|
||||
</SettingsSection>
|
||||
|
||||
{/* Desktop shell only - renders nothing in the browser/PWA build. */}
|
||||
<LocalIndexSettings />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
"use client";
|
||||
|
||||
// Settings panel for the desktop shell's encrypted local search index.
|
||||
//
|
||||
// Deliberately small: the index's PRIMARY trigger is the live push connection
|
||||
// (see lib/mail-index-client.ts's indexOnStateChange, wired into
|
||||
// stores/email-store.ts's handleStateChange), so this panel is a status readout
|
||||
// plus a manual catch-up button - not the mechanism.
|
||||
//
|
||||
// Renders nothing at all outside the Electron shell, where the routes 404.
|
||||
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { SettingsSection, SettingItem } from './settings-section';
|
||||
import { isElectronShell } from '@/lib/electron-bridge';
|
||||
import { useAccountStore } from '@/stores/account-store';
|
||||
import { catchUpIndex, fetchIndexStats, type IndexStats } from '@/lib/mail-index-client';
|
||||
import {
|
||||
chainSync, fetchReplicaStatus, purgeReplica, updateRetentionPolicy,
|
||||
type ReplicaStatus, type RetentionPolicy,
|
||||
} from '@/lib/offline-replica-client';
|
||||
|
||||
const TYPE_LABELS: Record<string, string> = {
|
||||
mail: 'Mail',
|
||||
calendar: 'Calendar',
|
||||
contact: 'Contacts',
|
||||
file: 'Files',
|
||||
};
|
||||
|
||||
export function LocalIndexSettings() {
|
||||
const slot = useAccountStore((s) => s.accounts.find((a) => a.id === s.activeAccountId)?.cookieSlot);
|
||||
const [stats, setStats] = useState<IndexStats[] | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [message, setMessage] = useState<string | null>(null);
|
||||
// `null` until the first probe resolves, so we don't flash a panel that then
|
||||
// vanishes on a non-desktop build.
|
||||
const [available, setAvailable] = useState<boolean | null>(null);
|
||||
|
||||
const refreshStats = useCallback(async () => {
|
||||
const next = await fetchIndexStats(slot);
|
||||
setStats(next);
|
||||
setAvailable(next !== null);
|
||||
}, [slot]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isElectronShell()) {
|
||||
setAvailable(false);
|
||||
return;
|
||||
}
|
||||
void refreshStats();
|
||||
}, [refreshStats]);
|
||||
|
||||
const handleRebuild = async () => {
|
||||
setBusy(true);
|
||||
setMessage(null);
|
||||
try {
|
||||
const result = await catchUpIndex(slot);
|
||||
if (result.unavailable) {
|
||||
setAvailable(false);
|
||||
setMessage(result.error ?? 'The encrypted index is unavailable on this system.');
|
||||
return;
|
||||
}
|
||||
if (!result.ok) {
|
||||
setMessage(result.error ?? 'Indexing failed.');
|
||||
return;
|
||||
}
|
||||
const written = Object.entries(result.written ?? {})
|
||||
.map(([type, n]) => `${TYPE_LABELS[type] ?? type}: ${n}`)
|
||||
.join(', ');
|
||||
const failed = (result.errors ?? []).map((e) => `${e.contentType} (${e.message})`).join('; ');
|
||||
setMessage(
|
||||
[
|
||||
written ? `Indexed ${written}.` : 'Nothing to index.',
|
||||
result.skipped?.length ? `Not supported: ${result.skipped.join(', ')}.` : '',
|
||||
failed ? `Problems: ${failed}` : '',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' '),
|
||||
);
|
||||
await refreshStats();
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (available === false || available === null) return null;
|
||||
|
||||
const total = (stats ?? []).reduce((sum, s) => sum + s.count, 0);
|
||||
|
||||
return (
|
||||
<SettingsSection
|
||||
title="Local search index"
|
||||
description={
|
||||
'An encrypted index of your recent mail, calendar events, contacts and file names, ' +
|
||||
'stored on this device only. It updates automatically as items arrive, and powers ' +
|
||||
'local search and AI answers about your own data. Files are indexed by name and ' +
|
||||
'location, not by their contents.'
|
||||
}
|
||||
>
|
||||
<SettingItem
|
||||
label="Indexed items"
|
||||
description={
|
||||
total > 0
|
||||
? (stats ?? [])
|
||||
.map((s) => `${TYPE_LABELS[s.contentType] ?? s.contentType}: ${s.count}`)
|
||||
.join(' · ')
|
||||
: 'Nothing indexed yet.'
|
||||
}
|
||||
>
|
||||
<span className="text-sm text-muted-foreground tabular-nums">{total}</span>
|
||||
</SettingItem>
|
||||
|
||||
<SettingItem
|
||||
label="Update now"
|
||||
description={
|
||||
message ??
|
||||
'Catches up on anything that changed while the app was closed. Normally not needed - ' +
|
||||
'the index updates itself when mail, events, contacts or files change.'
|
||||
}
|
||||
>
|
||||
<Button variant="outline" size="sm" onClick={handleRebuild} disabled={busy}>
|
||||
{busy ? 'Indexing…' : 'Update index'}
|
||||
</Button>
|
||||
</SettingItem>
|
||||
|
||||
<OfflineMailSettings slot={slot} />
|
||||
</SettingsSection>
|
||||
);
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
const units = ['KB', 'MB', 'GB'];
|
||||
let value = bytes / 1024;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit++; }
|
||||
return `${value.toFixed(value >= 10 ? 0 : 1)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
const PHASE_LABELS: Record<string, string> = {
|
||||
'never-run': 'not started',
|
||||
scanning: 'downloading history',
|
||||
reconciling: 'rebuilding',
|
||||
complete: 'up to date',
|
||||
};
|
||||
|
||||
/**
|
||||
* Controls for the offline mail replica (lib/offline-replica/**).
|
||||
*
|
||||
* Lives inside the same panel as the search index because they share one
|
||||
* encrypted file, one key and one purge - presenting them as two unrelated
|
||||
* features would misrepresent what "delete" deletes.
|
||||
*/
|
||||
function OfflineMailSettings({ slot }: { slot: number | undefined }) {
|
||||
const [status, setStatus] = useState<ReplicaStatus | null>(null);
|
||||
const [busy, setBusy] = useState<null | 'sync' | 'purge' | 'policy'>(null);
|
||||
const [message, setMessage] = useState<string | null>(null);
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
setStatus(await fetchReplicaStatus(slot));
|
||||
}, [slot]);
|
||||
|
||||
useEffect(() => { void refresh(); }, [refresh]);
|
||||
|
||||
const savePolicy = async (patch: Partial<RetentionPolicy>) => {
|
||||
if (!status) return;
|
||||
const next: RetentionPolicy = { ...status.policy, ...patch };
|
||||
setBusy('policy');
|
||||
setMessage(null);
|
||||
try {
|
||||
const ok = await updateRetentionPolicy(next, slot);
|
||||
if (!ok) { setMessage('Could not save the retention setting.'); return; }
|
||||
// The change is applied by the next cycle - a widen re-scans, a narrow
|
||||
// evicts - so run one now rather than leaving the number looking wrong.
|
||||
await chainSync({ slot, max: 2 });
|
||||
await refresh();
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handleSync = async () => {
|
||||
setBusy('sync');
|
||||
setMessage(null);
|
||||
try {
|
||||
const report = await chainSync({ slot });
|
||||
if (!report) { setMessage('Offline mail is unavailable on this system.'); return; }
|
||||
setMessage(
|
||||
report.ok
|
||||
? `Synced ${report.envelopesWritten} messages and ${report.bodiesWritten} bodies.` +
|
||||
(report.unfinishedWork ? ' More will download in the background.' : '') +
|
||||
(report.warnings.length > 0 ? ` Notes: ${report.warnings.join('; ')}` : '')
|
||||
: `Sync failed: ${report.error ?? 'unknown error'}`,
|
||||
);
|
||||
await refresh();
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handlePurge = async () => {
|
||||
setBusy('purge');
|
||||
setMessage(null);
|
||||
try {
|
||||
const ok = await purgeReplica(slot);
|
||||
setMessage(ok ? 'Offline mail deleted from this device.' : 'Could not delete offline mail.');
|
||||
await refresh();
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
};
|
||||
|
||||
if (!status) return null;
|
||||
|
||||
const stats = status.stats;
|
||||
const total = stats ? stats.fileBytes : 0;
|
||||
|
||||
return (
|
||||
<>
|
||||
<SettingItem
|
||||
label="Offline mail"
|
||||
description={
|
||||
stats
|
||||
? `${stats.envelopes} messages listed, ${stats.bodies} readable offline · ` +
|
||||
`${formatBytes(stats.bodyBytes)} of message content · ` +
|
||||
`status: ${PHASE_LABELS[status.coveragePhase] ?? status.coveragePhase}` +
|
||||
(status.resyncRequired ? ' (a rebuild is queued)' : '') +
|
||||
(stats.wantedBodies > 0 ? ` · ${stats.wantedBodies} still downloading` : '')
|
||||
: 'Nothing stored yet. Mail downloads automatically as it arrives.'
|
||||
}
|
||||
>
|
||||
<span className="text-sm text-muted-foreground tabular-nums">{formatBytes(total)}</span>
|
||||
</SettingItem>
|
||||
|
||||
<SettingItem
|
||||
label="Keep message list for"
|
||||
description={
|
||||
'How far back the offline message list goes. Listed messages are tiny (about a ' +
|
||||
'kilobyte each), so a wide window here costs very little and means a message never ' +
|
||||
'disappears from the offline list just because its content was removed to save space.'
|
||||
}
|
||||
>
|
||||
<select
|
||||
className="h-9 rounded-md border border-border bg-background px-2 text-sm"
|
||||
value={status.policy.envelopeDays}
|
||||
disabled={busy !== null}
|
||||
onChange={(e) => void savePolicy({ envelopeDays: Number(e.target.value) })}
|
||||
>
|
||||
{[30, 90, 180, 365, 730, 1825].map((d) => (
|
||||
<option key={d} value={d}>
|
||||
{d >= 365 ? `${Math.round(d / 365)} year${d >= 730 ? 's' : ''}` : `${d} days`}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</SettingItem>
|
||||
|
||||
<SettingItem
|
||||
label="Keep full messages for"
|
||||
description={
|
||||
'How far back complete messages - including formatted content - are stored so they ' +
|
||||
'can be read with no network. Attachments are not downloaded; they still need a ' +
|
||||
'connection.'
|
||||
}
|
||||
>
|
||||
<select
|
||||
className="h-9 rounded-md border border-border bg-background px-2 text-sm"
|
||||
value={status.policy.bodyDays}
|
||||
disabled={busy !== null}
|
||||
onChange={(e) => void savePolicy({ bodyDays: Number(e.target.value) })}
|
||||
>
|
||||
{[7, 14, 30, 90, 180, 365].map((d) => (
|
||||
<option key={d} value={d}>{d >= 365 ? '1 year' : `${d} days`}</option>
|
||||
))}
|
||||
</select>
|
||||
</SettingItem>
|
||||
|
||||
<SettingItem
|
||||
label="Storage limit for message content"
|
||||
description={
|
||||
'The oldest stored content is removed first when this is reached. Messages stay in ' +
|
||||
'the offline list either way - only their content is removed.'
|
||||
}
|
||||
>
|
||||
<select
|
||||
className="h-9 rounded-md border border-border bg-background px-2 text-sm"
|
||||
value={status.policy.maxBodyMB}
|
||||
disabled={busy !== null}
|
||||
onChange={(e) => void savePolicy({ maxBodyMB: Number(e.target.value) })}
|
||||
>
|
||||
{[100, 250, 500, 1000, 2000, 5000].map((mb) => (
|
||||
<option key={mb} value={mb}>{mb >= 1000 ? `${mb / 1000} GB` : `${mb} MB`}</option>
|
||||
))}
|
||||
</select>
|
||||
</SettingItem>
|
||||
|
||||
<SettingItem
|
||||
label="Offline mail actions"
|
||||
description={message ?? 'Download now, or delete everything stored offline on this device.'}
|
||||
>
|
||||
<div className="flex gap-2">
|
||||
<Button variant="outline" size="sm" onClick={handleSync} disabled={busy !== null}>
|
||||
{busy === 'sync' ? 'Downloading…' : 'Download now'}
|
||||
</Button>
|
||||
<Button variant="outline" size="sm" onClick={handlePurge} disabled={busy !== null}>
|
||||
{busy === 'purge' ? 'Deleting…' : 'Delete offline mail'}
|
||||
</Button>
|
||||
</div>
|
||||
</SettingItem>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# Registered on the dev-k8s-1/2/3 cluster (where ArgoCD already lives) via
|
||||
# `kubectl apply` directly to the argocd namespace — this file is the
|
||||
# version-controlled record of that, not something ArgoCD itself syncs
|
||||
# (no app-of-apps here, deliberately kept simple for two Applications).
|
||||
#
|
||||
# syncPolicy starts WITHOUT automated — manual sync until the one-time
|
||||
# per-namespace bootstrap (vncmail-env secret, image-pull secret — see
|
||||
# deploy/k8s/README.md §3) is done by hand once. Flip to automated (see
|
||||
# commented block below) only after a first manual sync succeeds cleanly.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: vncmail-dev
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@gitlab.vnc.biz:gitlab-instance-b9b5cf2f/vncmail-plus.git
|
||||
targetRevision: dev
|
||||
path: deploy/k8s/overlays/dev
|
||||
destination:
|
||||
server: https://kubernetes.default.svc # in-cluster — ArgoCD and vncmail-dev share this cluster
|
||||
namespace: vncmail
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
# automated:
|
||||
# prune: true
|
||||
# selfHeal: true
|
||||
@@ -0,0 +1,30 @@
|
||||
# NOT YET APPLIED to any cluster. Scaffolding only, matching
|
||||
# deploy/k8s/overlays/prod's own "inert until Phase D" status.
|
||||
#
|
||||
# Unlike vncmail-dev-app.yaml, this targets a DIFFERENT cluster (node1-3,
|
||||
# the HA "prod" cluster) than the one ArgoCD itself runs on (dev-k8s).
|
||||
# That means before this can be applied, node1-3 needs to be registered as
|
||||
# an ArgoCD-managed cluster (`argocd cluster add`, or an equivalent
|
||||
# ServiceAccount+kubeconfig secret) — deliberately not done yet: there's no
|
||||
# reason to wire cross-cluster RBAC into the prod HA cluster before prod
|
||||
# hostname/Stalwart/secrets are real and someone's actually promoting.
|
||||
#
|
||||
# syncPolicy has no automated block at all, and won't get one even later —
|
||||
# prod stays manual-sync-only permanently. That's the promotion gate.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: vncmail-prod
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: git@gitlab.vnc.biz:gitlab-instance-b9b5cf2f/vncmail-plus.git
|
||||
targetRevision: main
|
||||
path: deploy/k8s/overlays/prod
|
||||
destination:
|
||||
server: CHANGEME # the node1-3 cluster's registered ArgoCD server URL, once added
|
||||
namespace: vncmail-prod
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
+87
-38
@@ -1,29 +1,68 @@
|
||||
# VNCmail+ — Admin Deployment Guide (microk8s)
|
||||
|
||||
Deploy VNCmail+ (VNC's Bulwark fork) as a container at **`vncmail.sandbox.vnc.de`**,
|
||||
**alongside** the existing `bulwark.sandbox.vnc.de`. Plain `kubectl apply` — no
|
||||
GitOps needed.
|
||||
**alongside** the existing `bulwark.sandbox.vnc.de`.
|
||||
|
||||
> Why a container (not Vercel): Bulwark is stateful — it writes settings/admin/
|
||||
> telemetry to `/app/data`, which needs persistent volumes.
|
||||
|
||||
## Structure — base + overlays
|
||||
|
||||
```
|
||||
deploy/k8s/
|
||||
base/ # shared manifest shapes (namespace-agnostic)
|
||||
overlays/
|
||||
dev/ # the live sandbox — vncmail.sandbox.vnc.de, namespace vncmail
|
||||
prod/ # scaffolded, NOT YET LIVE — see "Production status" below
|
||||
ca/ # separate, isolated EJBCA internal CA — see ca/README.md.
|
||||
# Never composed with base/ or either overlay above.
|
||||
```
|
||||
|
||||
`kubectl apply -k overlays/dev` (or `overlays/prod`, once real) instead of
|
||||
applying `base/` directly — `base/` alone has no namespace and won't apply
|
||||
meaningfully on its own.
|
||||
|
||||
## Routine deploys go through CI + ArgoCD now
|
||||
|
||||
As of the GitLab CI/CD pipeline (`.gitlab-ci.yml`, see `../../VNCMAIL-SETUP.md`
|
||||
§ CI/CD), **pushing to `dev` auto-builds and bumps the deploy tag; ArgoCD's
|
||||
`vncmail-dev` Application applies it** — you should not normally need to run
|
||||
`kubectl apply` for the sandbox by hand anymore, and CI never touches the
|
||||
cluster directly (it only ever talks to the registry and to this git repo).
|
||||
This guide's manual steps below are for first-time setup, the one-time
|
||||
secret creation CI/ArgoCD deliberately never automate, and troubleshooting.
|
||||
|
||||
## Production status
|
||||
|
||||
**There is no production VNCmail+ deployment yet.** `overlays/prod/` exists
|
||||
in the repo but is inert: its ingress hostname and its secret's
|
||||
`JMAP_SERVER_URL` are both obvious placeholders (`vncmail.CHANGEME.invalid` /
|
||||
`https://REPLACE-ME-prod-stalwart-not-yet-deployed.invalid`) that will fail
|
||||
loudly rather than silently deploy against the wrong backend. Applying it
|
||||
requires, in order: a real prod Stalwart instance to exist, a real hostname
|
||||
decision, DNS, a real `secret.yaml`, and the `.gitlab-ci.yml` `promote` job's
|
||||
`kubectl apply` step (currently a TODO placeholder) filled in. None of that
|
||||
is CI's job to decide — it's an explicit, human-triggered event.
|
||||
|
||||
---
|
||||
|
||||
## 1. What you are deploying
|
||||
## 1. What you are deploying (per overlay)
|
||||
|
||||
| # | Object | File | Purpose |
|
||||
|---|--------|------|---------|
|
||||
| 1 | Namespace `vncmail` | `namespace.yaml` | Isolates the app |
|
||||
| 2 | 4× PersistentVolumeClaim | `pvc.yaml` | `/app/data/{settings,admin,admin-state,telemetry}` |
|
||||
| 3 | Secret `vncmail-env` | `secret.yaml` *(you create it)* | App config (JMAP URL, session secret, branding) |
|
||||
| 4 | Secret `ghcr-pull` | *(you create it — command below)* | Pull the private image from GHCR |
|
||||
| 5 | Deployment `vncmail-plus` | `deployment.yaml` | The app pod |
|
||||
| 6 | Service `vncmail-plus` | `service.yaml` | ClusterIP :80 → pod :3000 |
|
||||
| 7 | Ingress `vncmail-plus` | `ingress.yaml` | TLS host `vncmail.sandbox.vnc.de` |
|
||||
| 1 | Namespace | `overlays/<env>/namespace.yaml` | Isolates the app (`vncmail` for dev, `vncmail-prod` for prod) |
|
||||
| 2 | 4× PersistentVolumeClaim | `base/pvc.yaml` | `/app/data/{settings,admin,admin-state,telemetry}` |
|
||||
| 3 | Secret `vncmail-env` | `overlays/<env>/secret.yaml` *(you create it)* | App config (JMAP URL, session secret, branding) |
|
||||
| 4 | Image-pull secret | *(you create it — command below)* | Pull the (currently private) image |
|
||||
| 5 | Deployment `vncmail-plus` | `base/deployment.yaml` (+ overlay patches) | The app pod |
|
||||
| 6 | Service `vncmail-plus` | `base/service.yaml` | ClusterIP :80 → pod :3000 |
|
||||
| 7 | Ingress `vncmail-plus` | `base/ingress.yaml` (+ overlay patches for prod) | TLS host |
|
||||
|
||||
**Image:** `ghcr.io/brvncde-dotcom/vncmail-plus-dev:latest`
|
||||
(built automatically by CI from the `dev` branch). For anything beyond the
|
||||
sandbox, pin a digest — see §5.
|
||||
**Image:** CI builds and pushes to `registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus`
|
||||
(tag `sha-<sha>` per deploy, moving pointers `dev-latest`/`prod-latest`). The
|
||||
`ghcr.io/brvncde-dotcom/vncmail-plus-dev` image referenced in `base/deployment.yaml`
|
||||
is a legacy default only — CI overrides it per-deploy via `kubectl set image`,
|
||||
so what's committed there never needs to track what's actually running.
|
||||
|
||||
---
|
||||
|
||||
@@ -43,45 +82,48 @@ kubectl get ingressclass
|
||||
kubectl get clusterissuer # cert-manager issuers (if used)
|
||||
```
|
||||
|
||||
Then edit if they differ from the defaults below:
|
||||
Then edit if they differ from the defaults below (in `base/`, so both overlays
|
||||
pick up the fix):
|
||||
|
||||
| Value | Default in manifests | File to edit |
|
||||
|-------|----------------------|--------------|
|
||||
| StorageClass | `microk8s-hostpath` | `pvc.yaml` (all 4) |
|
||||
| IngressClass | `public` | `ingress.yaml` |
|
||||
| cert-manager issuer | `letsencrypt-prod` | `ingress.yaml` |
|
||||
| StorageClass | `microk8s-hostpath` | `base/pvc.yaml` (all 4) |
|
||||
| IngressClass | `public` | `base/ingress.yaml` |
|
||||
| cert-manager issuer | `letsencrypt-prod` | `base/ingress.yaml` |
|
||||
|
||||
---
|
||||
|
||||
## 3. Deploy (copy-paste, in order)
|
||||
## 3. First-time setup (one-time, per environment — CI never does this)
|
||||
|
||||
```bash
|
||||
cd deploy/k8s
|
||||
cd deploy/k8s/overlays/dev # or overlays/prod, once real
|
||||
|
||||
# a) Namespace
|
||||
kubectl apply -f namespace.yaml
|
||||
|
||||
# b) Image-pull secret — the GHCR package is private.
|
||||
# Use a GitHub PAT (classic) with the read:packages scope.
|
||||
# a) Image-pull secret — the registry package is private.
|
||||
kubectl create secret docker-registry ghcr-pull \
|
||||
--namespace vncmail \
|
||||
--docker-server=ghcr.io \
|
||||
--docker-username=brvncde-dotcom \
|
||||
--docker-password='<GITHUB_PAT_read:packages>' \
|
||||
--docker-email=br@vnc.biz
|
||||
# Once CI has cut over to registry.gitlab.vnc.biz, this becomes a
|
||||
# docker-registry secret for that registry instead — see VNCMAIL-SETUP.md.
|
||||
|
||||
# c) App config secret — copy the template, set a real SESSION_SECRET, apply.
|
||||
# b) App config secret — copy the template, set a real SESSION_SECRET, apply.
|
||||
cp secret.example.yaml secret.yaml
|
||||
# edit secret.yaml: SESSION_SECRET: "$(openssl rand -base64 32)"
|
||||
kubectl apply -f secret.yaml
|
||||
|
||||
# d) Everything else (PVCs, Deployment, Service, Ingress)
|
||||
# c) Everything else (namespace, PVCs, Deployment, Service, Ingress)
|
||||
kubectl apply -k .
|
||||
```
|
||||
|
||||
> Alternative to (b): make the GHCR package public
|
||||
> (GitHub → Packages → vncmail-plus-dev → Package settings → Change visibility),
|
||||
> then delete the `imagePullSecrets:` block from `deployment.yaml`.
|
||||
> Alternative to (a): make the registry package public, then delete the
|
||||
> `imagePullSecrets:` block from `base/deployment.yaml`.
|
||||
|
||||
After this one-time setup, routine deploys to `dev` happen automatically via
|
||||
CI on every push — see "Routine deploys go through CI now" above. This
|
||||
section is for first-time bring-up (or `overlays/prod`, once it's real) and
|
||||
troubleshooting, not the everyday path.
|
||||
|
||||
---
|
||||
|
||||
@@ -104,16 +146,23 @@ a bare username.
|
||||
|
||||
## 5. Update to a new build
|
||||
|
||||
```bash
|
||||
# CI rebuilds ghcr.io/brvncde-dotcom/vncmail-plus-dev on every push to `dev`.
|
||||
kubectl -n vncmail rollout restart deploy/vncmail-plus # pulls :latest (imagePullPolicy: Always)
|
||||
Normally you don't — CI's `bump-dev` job + ArgoCD's automated sync do this
|
||||
on every push to `dev`. To do it by hand (e.g. troubleshooting, before
|
||||
automated sync is turned on):
|
||||
|
||||
# Production: pin a digest instead of :latest so rollouts are deterministic.
|
||||
```bash
|
||||
kubectl -n vncmail set image deploy/vncmail-plus \
|
||||
vncmail-plus=ghcr.io/brvncde-dotcom/vncmail-plus-dev@sha256:<digest>
|
||||
vncmail-plus=registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus:sha-<sha>
|
||||
```
|
||||
|
||||
Rollback: `kubectl -n vncmail rollout undo deploy/vncmail-plus`
|
||||
ArgoCD will overwrite this on its next sync unless you also update
|
||||
`deploy/k8s/overlays/dev/image-tag/kustomization.yaml` to match — that file
|
||||
is CI-owned (see its header comment), so a by-hand `set image` is only ever
|
||||
a temporary override, not a real fix.
|
||||
|
||||
Rollback (bypassing ArgoCD temporarily): `kubectl -n vncmail rollout undo deploy/vncmail-plus`.
|
||||
The real rollback is reverting the commit that bumped the tag and letting
|
||||
ArgoCD re-sync.
|
||||
|
||||
---
|
||||
|
||||
@@ -121,9 +170,9 @@ Rollback: `kubectl -n vncmail rollout undo deploy/vncmail-plus`
|
||||
|
||||
| Symptom | Cause / fix |
|
||||
|---------|-------------|
|
||||
| Pod `ImagePullBackOff` | `ghcr-pull` secret missing/expired, or package still private. Recreate the secret (§3b) or make the package public. |
|
||||
| Pod `CrashLoopBackOff`, logs show `EACCES`/permission on `/app/data` | Volume not writable by uid 1001. `securityContext.fsGroup: 1001` is set in `deployment.yaml` — keep it; some storage drivers also need it on the PVC. |
|
||||
| PVC stuck `Pending` | Wrong `storageClassName` in `pvc.yaml`. Set it to one from `kubectl get sc`. |
|
||||
| Pod `ImagePullBackOff` | `ghcr-pull` secret missing/expired, or package still private. Recreate the secret (§3a) or make the package public. |
|
||||
| Pod `CrashLoopBackOff`, logs show `EACCES`/permission on `/app/data` | Volume not writable by uid 1001. `securityContext.fsGroup: 1001` is set in `base/deployment.yaml` — keep it; some storage drivers also need it on the PVC. |
|
||||
| PVC stuck `Pending` | Wrong `storageClassName` in `base/pvc.yaml`. Set it to one from `kubectl get sc`. |
|
||||
| Ingress has no address / no cert | Wrong `ingressClassName` or cert issuer. Match bulwark's (§2). Check `kubectl -n vncmail describe ingress vncmail-plus`. |
|
||||
| Login shows "Ein Fehler ist aufgetreten" | Use the **full** email (`user@sandbox.vnc.de`), not a bare username. |
|
||||
| Can't reach Stalwart | Check `JMAP_SERVER_URL` in the secret = `https://stalwart.sandbox.vnc.de`. |
|
||||
|
||||
@@ -2,7 +2,6 @@ apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
namespace: vncmail
|
||||
labels:
|
||||
app: vncmail-plus
|
||||
spec:
|
||||
@@ -24,14 +23,19 @@ spec:
|
||||
fsGroup: 1001
|
||||
runAsUser: 1001
|
||||
runAsGroup: 1001
|
||||
# ghcr package is private by default — see deploy/k8s/README.md to create
|
||||
# this pull secret. Delete this block if you make the package public.
|
||||
imagePullSecrets:
|
||||
- name: ghcr-pull
|
||||
# Confirmed 2026-08-05: ghcr.io/brvncde-dotcom/vncmail-plus-dev IS public
|
||||
# (anonymous token pull succeeded) — no imagePullSecrets needed. This is
|
||||
# deploy/k8s/README.md's own documented alternative to creating a
|
||||
# ghcr-pull secret. Removed rather than left referencing a
|
||||
# not-yet-created secret, which would otherwise block every pod from
|
||||
# starting regardless of the image being public (kubelet fails to
|
||||
# resolve a missing imagePullSecrets entry before it ever gets to
|
||||
# deciding whether auth was actually required).
|
||||
containers:
|
||||
- name: vncmail-plus
|
||||
# dev image (built from the `dev` branch by CI). For production pin a
|
||||
# digest: ghcr.io/brvncde-dotcom/vncmail-plus-dev@sha256:<digest>
|
||||
# Default/legacy value — CI overrides the image per-deploy via
|
||||
# `kustomize edit set image`, so what's committed here never goes
|
||||
# stale. For a one-off manual apply, pin a digest instead of :latest.
|
||||
image: ghcr.io/brvncde-dotcom/vncmail-plus-dev:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
@@ -0,0 +1,34 @@
|
||||
# Both real clusters (node1-3 "prod", dev-k8s-1-3 "dev") run Traefik, not
|
||||
# nginx — confirmed via `kubectl get ingressclass` (class is literally named
|
||||
# `traefik`). Unlike nginx's restrictive 1MB default, Traefik has no default
|
||||
# request-body-size cap, so there's no equivalent needed for mail attachment
|
||||
# uploads (the old nginx.ingress.kubernetes.io/proxy-body-size annotation
|
||||
# this file used to carry is simply not applicable here).
|
||||
#
|
||||
# Host, TLS secretName, and cert-manager issuer are ALL overlay-specific now
|
||||
# (dev-k8s only has a `letsencrypt-staging` issuer; node1-3/prod has none
|
||||
# configured yet) — every overlay's patch-ingress.yaml must override the
|
||||
# CHANGEME placeholders below.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: CHANGEME
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- CHANGEME.invalid
|
||||
secretName: vncmail-plus-tls
|
||||
rules:
|
||||
- host: CHANGEME.invalid
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: vncmail-plus
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- ingress.yaml
|
||||
# - secret.yaml # create from an overlay's secret.example.yaml; not committed
|
||||
|
||||
# Namespace is intentionally NOT set here. Kustomize's `namespace:` transformer
|
||||
# doesn't rename cluster-scoped Namespace objects, so each overlay ships its own
|
||||
# namespace.yaml (the actual object) and its own `namespace:` field (which
|
||||
# injects metadata.namespace into every namespaced resource below). Applying
|
||||
# this base directly is meaningless — always go through an overlay.
|
||||
@@ -5,7 +5,6 @@ apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vncmail-settings
|
||||
namespace: vncmail
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: microk8s-hostpath
|
||||
@@ -17,7 +16,6 @@ apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vncmail-admin
|
||||
namespace: vncmail
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: microk8s-hostpath
|
||||
@@ -29,7 +27,6 @@ apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vncmail-admin-state
|
||||
namespace: vncmail
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: microk8s-hostpath
|
||||
@@ -41,7 +38,6 @@ apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: vncmail-telemetry
|
||||
namespace: vncmail
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: microk8s-hostpath
|
||||
@@ -2,7 +2,6 @@ apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
namespace: vncmail
|
||||
labels:
|
||||
app: vncmail-plus
|
||||
spec:
|
||||
@@ -0,0 +1,479 @@
|
||||
# VNC internal CA — EJBCA Community on microk8s
|
||||
|
||||
Runbook for `A-01` / `A-06`. Issues 1-year S/MIME certificates for VNCmail+.
|
||||
|
||||
You run every command here. Claude wrote the manifests and cannot reach the
|
||||
cluster (no kubeconfig on the authoring machine), and the root-key ceremony in
|
||||
§3 **must not** be automated by an agent — the entire value of an offline root is
|
||||
that its private key never exists on a machine that runs services or tooling.
|
||||
|
||||
---
|
||||
|
||||
## 0. One decision to make before you type anything
|
||||
|
||||
**Name the root for the organisation, not the environment.**
|
||||
|
||||
You asked for sandbox first with the ability to promote to `vncmail` at any time.
|
||||
The way that stays cheap is a single root, generated once, with *per-environment
|
||||
intermediates* underneath it:
|
||||
|
||||
```
|
||||
VNC Root CA R1 offline · 15y · RSA 4096 · pathlen:1
|
||||
├─ VNC S/MIME Issuing CA Sandbox R1 in-cluster · 5y · RSA 4096 · pathlen:0 → *@sandbox.vnc.de
|
||||
└─ VNC S/MIME Issuing CA R1 in-cluster · 5y · RSA 4096 · pathlen:0 → *@vncmail.de (later)
|
||||
```
|
||||
|
||||
Promotion is then "issue a second intermediate from the same root" — a one-hour
|
||||
ceremony. The trust anchor you distribute to laptops, phones and partners does
|
||||
not change, and certificates already issued keep validating.
|
||||
|
||||
The alternative — a throwaway `VNC Sandbox Root` — means that on promotion you
|
||||
redistribute a new trust anchor to every device and every external party who
|
||||
ever verified one of your signatures. That is the expensive path, and it is only
|
||||
visible as expensive later.
|
||||
|
||||
So: **generate the root at prod grade, once, now**, even though the first
|
||||
intermediate only serves `@sandbox.vnc.de`. The extra cost today is choosing a
|
||||
better passphrase and a safe to keep the USB key in.
|
||||
|
||||
> RSA 4096 rather than an elliptic curve throughout, deliberately. ECDSA S/MIME
|
||||
> is still poorly handled by older Outlook and by several mobile clients, and
|
||||
> S/MIME interop failures are silent — the recipient sees a broken signature, not
|
||||
> an error you get told about. Pay the key-size cost for interop you can't test.
|
||||
|
||||
---
|
||||
|
||||
## 1. Install
|
||||
|
||||
```bash
|
||||
kubectl apply -f deploy/k8s/ca/namespace.yaml
|
||||
```
|
||||
|
||||
Fill in and apply the secret out-of-band (never commit real values):
|
||||
|
||||
```bash
|
||||
cp deploy/k8s/ca/secret.example.yaml /tmp/ca-secret.yaml && $EDITOR /tmp/ca-secret.yaml
|
||||
```
|
||||
|
||||
```bash
|
||||
kubectl apply -f /tmp/ca-secret.yaml && shred -u /tmp/ca-secret.yaml
|
||||
```
|
||||
|
||||
```bash
|
||||
kubectl apply -k deploy/k8s/ca/
|
||||
```
|
||||
|
||||
First boot builds the EJBCA schema and takes several minutes. Watch it rather
|
||||
than assuming it hung:
|
||||
|
||||
```bash
|
||||
kubectl -n vnc-ca logs -f deploy/ejbca
|
||||
```
|
||||
|
||||
```bash
|
||||
kubectl -n vnc-ca get pods -w
|
||||
```
|
||||
|
||||
### Verify before going further
|
||||
|
||||
```bash
|
||||
kubectl -n vnc-ca exec deploy/ejbca -- curl -sf http://localhost:8080/ejbca/publicweb/healthcheck/ejbcahealth && echo OK
|
||||
```
|
||||
|
||||
If the manifests' env-var names have drifted from the image tag you pulled, this
|
||||
is where it shows up — EJBCA will start but fail to bind its datasource. Check
|
||||
the documented variables for your tag before editing anything else:
|
||||
|
||||
```bash
|
||||
kubectl -n vnc-ca logs deploy/ejbca | grep -iE "datasource|jdbc|database"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Get administrative access
|
||||
|
||||
EJBCA's admin web requires a client certificate. On first boot the container
|
||||
enrols a `SuperAdmin` and writes a PKCS#12 inside the pod.
|
||||
|
||||
```bash
|
||||
kubectl -n vnc-ca exec deploy/ejbca -- find / -name "*.p12" -newermt "-1 day" 2>/dev/null
|
||||
```
|
||||
|
||||
Copy it out, import it into your browser, then reach the admin web by
|
||||
port-forward — it is not exposed through any ingress and must not be:
|
||||
|
||||
```bash
|
||||
kubectl -n vnc-ca port-forward deploy/ejbca 8443:8443
|
||||
```
|
||||
|
||||
Then open `https://localhost:8443/ejbca/adminweb`.
|
||||
|
||||
> If the container did not create a SuperAdmin (behaviour differs by tag), use
|
||||
> the CLI inside the pod instead:
|
||||
> `kubectl -n vnc-ca exec -it deploy/ejbca -- /opt/keyfactor/bin/ejbca.sh ra addendentity ...`
|
||||
> followed by `setclearpwd` and a browser enrolment against
|
||||
> `https://localhost:8443/ejbca/ra/`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Root ceremony — you, offline, once
|
||||
|
||||
Do this on a machine that is **not** this cluster and **not** your daily laptop
|
||||
if you can manage it. A live USB session on a machine with networking physically
|
||||
off is enough for a sandbox-grade start; the point is that the root key never
|
||||
touches a host that runs services.
|
||||
|
||||
Everything below happens in one directory that you will destroy at the end.
|
||||
|
||||
**3.1 Prepare the config.** Save as `root.cnf`:
|
||||
|
||||
```ini
|
||||
[ req ]
|
||||
default_md = sha256
|
||||
prompt = no
|
||||
distinguished_name = dn
|
||||
x509_extensions = root_ext
|
||||
|
||||
[ dn ]
|
||||
C = CH
|
||||
O = VNC AG
|
||||
CN = VNC Root CA R1
|
||||
|
||||
[ root_ext ]
|
||||
basicConstraints = critical,CA:TRUE,pathlen:1
|
||||
keyUsage = critical,keyCertSign,cRLSign
|
||||
subjectKeyIdentifier = hash
|
||||
|
||||
# --- used in 3.4 to sign the intermediate CSR ---
|
||||
[ ca ]
|
||||
default_ca = CA_root
|
||||
|
||||
[ CA_root ]
|
||||
new_certs_dir = .
|
||||
database = index.txt
|
||||
serial = serial
|
||||
private_key = root.key
|
||||
certificate = root.crt
|
||||
default_md = sha256
|
||||
policy = policy_any
|
||||
crl = root.crl
|
||||
default_crl_days = 365
|
||||
unique_subject = no
|
||||
|
||||
[ policy_any ]
|
||||
countryName = optional
|
||||
organizationName = optional
|
||||
organizationalUnitName = optional
|
||||
commonName = supplied
|
||||
|
||||
[ int_ext ]
|
||||
basicConstraints = critical,CA:TRUE,pathlen:0
|
||||
keyUsage = critical,keyCertSign,cRLSign
|
||||
subjectKeyIdentifier = hash
|
||||
authorityKeyIdentifier = keyid:always
|
||||
# Revocation pointers for the INTERMEDIATE itself, served by the root's CRL.
|
||||
crlDistributionPoints = URI:http://ca.sandbox.vnc.de/ejbca/publicweb/crls/root.crl
|
||||
```
|
||||
|
||||
`pathlen:1` on the root and `pathlen:0` on the intermediate together mean the
|
||||
intermediate can issue end-entity certificates and nothing else. It cannot mint
|
||||
a further CA even if its key is stolen — that limits a compromise to "revoke one
|
||||
intermediate" instead of "the whole hierarchy is untrustworthy".
|
||||
|
||||
**3.2 Generate the root key.** You will be asked for a passphrase. Generate it
|
||||
with a password manager, minimum 24 random characters, and record where it lives
|
||||
*before* you type it — a root key whose passphrase is lost is a hierarchy you
|
||||
have to rebuild.
|
||||
|
||||
```bash
|
||||
openssl genrsa -aes256 -out root.key 4096
|
||||
```
|
||||
|
||||
**3.3 Self-sign the root.** 15 years, so it outlives several intermediate
|
||||
rotations and you do the ceremony once:
|
||||
|
||||
```bash
|
||||
openssl req -new -x509 -config root.cnf -key root.key -sha256 -days 5480 -out root.crt
|
||||
```
|
||||
|
||||
```bash
|
||||
openssl x509 -in root.crt -noout -text | sed -n '1,25p'
|
||||
```
|
||||
|
||||
Confirm in that output: `CA:TRUE, pathlen:1`, `Key Usage: Certificate Sign, CRL Sign`,
|
||||
and a 15-year validity window. If `basicConstraints` is missing the root is
|
||||
useless — the config's `x509_extensions` did not apply.
|
||||
|
||||
**3.4 Sign the intermediate.** EJBCA generates the intermediate key *inside the
|
||||
cluster* and hands you a CSR; the intermediate's private key never leaves EJBCA
|
||||
and never appears in this directory.
|
||||
|
||||
In the admin web: **CA Functions → Certificate Authorities → Create CA**
|
||||
- Name: `VNC S/MIME Issuing CA Sandbox R1`
|
||||
- Subject DN: `CN=VNC S/MIME Issuing CA Sandbox R1,O=VNC AG,C=CH`
|
||||
- Crypto Token: create a new soft token, PIN = `EJBCA_CRYPTO_TOKEN_PIN` from your secret
|
||||
- Key: RSA 4096, signing algorithm SHA256WithRSA
|
||||
- **Signed By: External CA** ← this is what makes it emit a CSR instead of self-signing
|
||||
- Validity: `5y`
|
||||
- CRL Expire Period: `1d`, CRL Overlap: `10m`
|
||||
- Default CRL Distribution Point: `http://ca.sandbox.vnc.de/ejbca/publicweb/crls/search.cgi?iHash=...` (EJBCA fills the hash — take what it offers)
|
||||
|
||||
Save, download the CSR, move it to the offline machine, then:
|
||||
|
||||
```bash
|
||||
touch index.txt && echo 1000 > serial
|
||||
```
|
||||
|
||||
```bash
|
||||
openssl ca -config root.cnf -extensions int_ext -days 1825 -notext -in sandbox-issuing.csr -out sandbox-issuing.crt
|
||||
```
|
||||
|
||||
**3.5 Issue the root CRL.** Do this now, in the same ceremony — not later. A root
|
||||
that has never published a CRL cannot revoke a compromised intermediate, and you
|
||||
will not want to bring the root key out under incident pressure just to
|
||||
discover the procedure doesn't work:
|
||||
|
||||
```bash
|
||||
openssl ca -config root.cnf -gencrl -out root.crl
|
||||
```
|
||||
|
||||
```bash
|
||||
openssl crl -in root.crl -noout -text | head -12
|
||||
```
|
||||
|
||||
**3.6 Take the outputs off, then destroy the directory.** Off the machine:
|
||||
`root.crt`, `root.crl`, `sandbox-issuing.crt`, and `root.key` (to encrypted
|
||||
storage, two copies, two physical locations).
|
||||
|
||||
```bash
|
||||
shred -u root.key && rm -rf ./*
|
||||
```
|
||||
|
||||
The root key comes out of the safe for exactly three reasons: signing a new
|
||||
intermediate (promotion to `vncmail.de`), refreshing the root CRL before it
|
||||
expires (annually — put it in a calendar now), or revoking an intermediate.
|
||||
|
||||
**3.7 Import the chain back into EJBCA.** Admin web → the CA you created →
|
||||
**Import CA certificate**, upload `root.crt` then `sandbox-issuing.crt`. The CA
|
||||
status must move to `Active`. Publish `root.crl` so the URL in the
|
||||
intermediate's CDP actually resolves.
|
||||
|
||||
---
|
||||
|
||||
## 4. Certificate profile — 1-year S/MIME
|
||||
|
||||
**Certificate Profiles → Add** → `VNC S/MIME 1y`, type *End Entity*.
|
||||
|
||||
| Setting | Value | Why |
|
||||
|---|---|---|
|
||||
| Validity | `1y` | your decision |
|
||||
| Key algorithms | RSA 2048, 3072, 4096 | 2048 floor for interop; no ECDSA yet (§0) |
|
||||
| Key Usage | `digitalSignature`, `keyEncipherment` | signing **and** decryption need both |
|
||||
| Extended Key Usage | `emailProtection` | critical — see below |
|
||||
| Subject Alternative Name | `rfc822Name`, **required** | this is the authoritative address |
|
||||
| Basic Constraints | CA:FALSE, critical | |
|
||||
| CRL Distribution Point | use CA default | |
|
||||
| OCSP Service Locator (AIA) | `http://ca.sandbox.vnc.de/ejbca/publicweb/status/ocsp` | |
|
||||
| Allow key recovery | **on** | see §7 |
|
||||
| Allow subject DN override by CSR | **OFF** | load-bearing, see below |
|
||||
| Allow extension override by CSR | **OFF** | load-bearing, see below |
|
||||
| Allow subject alt name override by CSR | **OFF** | load-bearing, see below |
|
||||
|
||||
**The three override settings must be OFF, and this is the single most important
|
||||
line in this document.**
|
||||
|
||||
The enrolment route deliberately does *not* inspect the CSR to police what it
|
||||
asks for. It doesn't need to: the route supplies the subject and the
|
||||
`rfc822Name` SAN itself, from addresses Stalwart confirmed the account may send
|
||||
from, and the CSR contributes only a public key plus proof the requester holds
|
||||
the matching private key.
|
||||
|
||||
That reasoning is only sound while EJBCA ignores the CSR's own subject and
|
||||
extensions. Turn any of these overrides on and a hand-crafted CSR claiming
|
||||
`rfc822Name=ceo@vnc.de` gets exactly that certificate — no code change, no
|
||||
alert, and the enrolment route still looks correct in review. It is a
|
||||
one-checkbox path from "authenticated users get certificates for their own
|
||||
addresses" to "authenticated users get certificates for anyone's address".
|
||||
|
||||
Verify it rather than trusting the profile screen, once the route is live:
|
||||
|
||||
```bash
|
||||
openssl req -new -key /tmp/t.key -subj "/CN=Impostor" -addext "subjectAltName=email:ceo@vnc.de" -out /tmp/t.csr
|
||||
```
|
||||
|
||||
Submit that CSR through the enrolment route as an ordinary user. The certificate
|
||||
that comes back must carry **your own** address, not `ceo@vnc.de`.
|
||||
|
||||
Two of these carry real weight:
|
||||
|
||||
**`emailProtection` EKU, and only that.** A certificate with no EKU is treated by
|
||||
some clients as valid for *anything* — TLS server auth included. Constrain it.
|
||||
|
||||
**`rfc822Name` SAN required.** Modern clients bind the sender address from the
|
||||
SAN, not the `emailAddress` DN attribute. Our forked plugin's fix-1 check
|
||||
(`signerEmailMatch`, which refuses to auto-import a signer cert whose address
|
||||
doesn't match the `From` header) now reads the address the same way clients do —
|
||||
SAN first, and matched against *every* address the certificate carries. If EJBCA
|
||||
issues certificates without an `rfc822Name` SAN, that check fails closed and
|
||||
encryption silently never becomes available.
|
||||
|
||||
Populating the DN `emailAddress` attribute as well, for old Outlook, is safe —
|
||||
but only as of finding 11. Until then the plugin read the DN attribute *in
|
||||
preference to* the SAN and compared only the first address it found, so an EJBCA
|
||||
certificate with both fields populated would have reported every genuine
|
||||
signature as "signer ≠ From" and blocked the import. Covered now by
|
||||
`vnc/plugins/smime/verify-address-binding.mjs`.
|
||||
|
||||
**End Entity Profiles → Add** → `VNC S/MIME User`:
|
||||
- Default Certificate Profile: `VNC S/MIME 1y`; available: the same only
|
||||
- Subject DN: `CN` required + modifiable, `O=VNC AG` and `C=CH` fixed
|
||||
- Subject Alt Name: `rfc822Name` required, **and tick "Use entity email field"**
|
||||
- Default CA: `VNC S/MIME Issuing CA Sandbox R1`
|
||||
|
||||
---
|
||||
|
||||
## 5. RA credential for the enrolment route
|
||||
|
||||
The webmail server — not the browser — calls the REST API. It needs its own
|
||||
client certificate with *only* the authority to enrol end entities.
|
||||
|
||||
**5.1** Create a certificate profile `VNC RA Client` (End Entity, EKU
|
||||
`clientAuth`, validity `1y`) and enrol one entity `CN=vncmail-ra-sandbox`
|
||||
against it. Download as PKCS#12.
|
||||
|
||||
**5.2** Restrict it. **System Functions → Administrator Roles → Add** →
|
||||
`VNCmail RA (sandbox)`:
|
||||
|
||||
| Rule | Access |
|
||||
|---|---|
|
||||
| `/ca_functionality/create_certificate` | Allow |
|
||||
| `/ca/VNC S/MIME Issuing CA Sandbox R1` | Allow |
|
||||
| `/endentityprofilesrules/VNC S/MIME User/**` | Allow |
|
||||
| `/ra_functionality/revoke_end_entity` | Allow |
|
||||
| everything else | **not granted** |
|
||||
|
||||
Match by the certificate's serial + issuer DN, not by CN. Do **not** give this
|
||||
role `/administrator` or any `/system_functionality` rule: this credential lives
|
||||
on an internet-facing pod, and the blast radius of it leaking should be "issue
|
||||
and revoke S/MIME certs under one profile", not "reconfigure the CA".
|
||||
|
||||
**5.3** Load it into the webmail namespace:
|
||||
|
||||
```bash
|
||||
kubectl -n vncmail create secret generic smime-ra \
|
||||
--from-file=client.p12=./vncmail-ra-sandbox.p12 \
|
||||
--from-literal=client-password='<p12 passphrase>' \
|
||||
--from-file=ca-chain.pem=./chain.pem
|
||||
```
|
||||
|
||||
`chain.pem` is `sandbox-issuing.crt` followed by `root.crt`. The enrolment route
|
||||
pins this chain when it connects to EJBCA on 8443 — it does not trust the public
|
||||
root store, so EJBCA's self-signed server certificate (`TLS_SETUP_ENABLED=simple`)
|
||||
is correct and expected here.
|
||||
|
||||
---
|
||||
|
||||
## 6. Verify the network policy actually enforces
|
||||
|
||||
Applying a NetworkPolicy on a CNI that doesn't implement it succeeds silently
|
||||
and protects nothing. Prove it:
|
||||
|
||||
```bash
|
||||
kubectl -n default run np-probe --rm -it --image=curlimages/curl --restart=Never -- \
|
||||
curl -sS -m 5 -k https://ejbca.vnc-ca.svc.cluster.local:8443/ejbca/ejbca-rest-api/v1/ca
|
||||
```
|
||||
|
||||
This **must** time out or be refused. If it returns anything HTTP-shaped —
|
||||
including a `401` — the policy is not being enforced and the REST API is exposed
|
||||
cluster-wide. Check your CNI before continuing:
|
||||
|
||||
```bash
|
||||
kubectl -n kube-system get pods | grep -iE "calico|cilium|flannel"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Key recovery is not optional here
|
||||
|
||||
S/MIME differs from TLS in a way that has bitten every organisation that
|
||||
deployed it without thinking about this: **if a user loses their private key,
|
||||
every message ever encrypted to them is permanently unreadable.** Not
|
||||
inconvenient — gone. Re-issuing a certificate does not help, because the old
|
||||
messages were encrypted to the old key.
|
||||
|
||||
So `Allow key recovery` in §4 is deliberate, and it is a real trade-off:
|
||||
|
||||
- **on** — EJBCA escrows the decryption key. Lost laptop is recoverable. But the
|
||||
CA database now contains material that decrypts users' mail, so §8 backup
|
||||
handling and the §5 role restrictions become load-bearing, and the escrow is
|
||||
something you must be able to explain to a user asking whether their mail is
|
||||
end-to-end encrypted. It is, from the wire's perspective; it is not, from the
|
||||
CA operator's.
|
||||
- **off** — nobody but the user can ever read their mail, and a lost device is
|
||||
permanent data loss with no recourse.
|
||||
|
||||
For a corporate deployment where mail is a business record, escrow on is the
|
||||
defensible choice, and it's what §4 sets. Decide this consciously — it is far
|
||||
cheaper to turn on now than to explain later why three years of mail is gone.
|
||||
|
||||
If you keep it on, use a separate key-recovery role with two-person approval
|
||||
rather than folding that authority into the RA credential.
|
||||
|
||||
---
|
||||
|
||||
## 8. Backup
|
||||
|
||||
`ejbca-db-data` contains the intermediate CA private key and — per §7 — escrowed
|
||||
user decryption keys. A dump of it is equivalent to the CA itself.
|
||||
|
||||
```bash
|
||||
kubectl -n vnc-ca exec deploy/ejbca-db -- sh -c \
|
||||
'mariadb-dump -u root -p"$MARIADB_ROOT_PASSWORD" --single-transaction ejbca' \
|
||||
| gzip > ejbca-$(date +%F).sql.gz
|
||||
```
|
||||
|
||||
Encrypt before it leaves your machine — an unencrypted CA dump in object storage
|
||||
is the whole hierarchy:
|
||||
|
||||
```bash
|
||||
gpg --symmetric --cipher-algo AES256 ejbca-$(date +%F).sql.gz
|
||||
```
|
||||
|
||||
Then to the shared R2 bucket (`vnc-backups1`) and **delete the plaintext**.
|
||||
Restore-test it once, now, against a scratch namespace — an untested CA backup is
|
||||
a belief, not a backup.
|
||||
|
||||
Not in this backup, by design and stored separately: the offline root key
|
||||
(§3.6), `EJBCA_CRYPTO_TOKEN_PIN`, and the RA PKCS#12 passphrase.
|
||||
|
||||
---
|
||||
|
||||
## 9. Promotion to production
|
||||
|
||||
Nothing here is thrown away. Same root, new intermediate:
|
||||
|
||||
1. Bring `root.key` out of the safe; repeat §3.4–3.6 for
|
||||
`CN=VNC S/MIME Issuing CA R1` — sign it with the **same root**.
|
||||
2. Duplicate the §4 profiles as `VNC S/MIME 1y (prod)` bound to the new CA.
|
||||
3. Fresh RA credential and role for the prod webmail namespace (§5). Never share
|
||||
the sandbox one across environments.
|
||||
4. Point the prod CDP/AIA at a stable production hostname. Those URLs are baked
|
||||
into every certificate for its full year, so get the hostname right *before*
|
||||
the first issuance.
|
||||
|
||||
The trust anchor on user devices does not change, and sandbox-issued
|
||||
certificates keep validating.
|
||||
|
||||
## 10. SwissSign (P7, deferred)
|
||||
|
||||
The point of the `CaProvider` interface on the application side is that this
|
||||
whole document becomes one implementation of it. Moving to SwissSign-issued
|
||||
certificates — for ZertES/eIDAS-qualified signatures that external parties
|
||||
validate without installing anything — is then a second implementation plus an
|
||||
identity-verification step, not a rewrite of the enrolment flow.
|
||||
|
||||
What survives unchanged: in-browser key generation, CSR construction, the
|
||||
enrolment route, storage, sign/encrypt/decrypt, the UI.
|
||||
What changes: who signs the CSR, and the fact that a human must prove their
|
||||
identity before a qualified certificate is issued — which is a process
|
||||
requirement, not a code one.
|
||||
@@ -0,0 +1,114 @@
|
||||
# EJBCA Community Edition.
|
||||
#
|
||||
# VERIFY THE ENV CONTRACT BEFORE YOU TRUST THIS FILE. EJBCA's container
|
||||
# configuration has changed across releases, so pin a tag and check its
|
||||
# documented variables rather than assuming these carry over:
|
||||
# docker run --rm keyfactor/ejbca-ce:<tag> cat /opt/keyfactor/bin/start.sh | head -60
|
||||
# The shape below (external MariaDB, two ports, healthcheck path) is stable; the
|
||||
# individual variable names are the part most likely to drift.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ejbca
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: ejbca
|
||||
ports:
|
||||
# 8080 — plain HTTP, NO client-certificate authentication. Only the public
|
||||
# web is served here: CRL distribution and the OCSP responder. This is the
|
||||
# only port the public ingress touches.
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
# 8443 — HTTPS with mandatory client-certificate auth. Admin web AND the
|
||||
# REST API. Never exposed through an ingress; reachable only from inside the
|
||||
# cluster (the enrolment route) or via `kubectl port-forward` (you, doing
|
||||
# administration). See networkpolicy.yaml.
|
||||
- name: https
|
||||
port: 8443
|
||||
targetPort: 8443
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ejbca
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: ejbca
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ejbca
|
||||
spec:
|
||||
# EJBCA needs the DB reachable before WildFly deploys its datasource.
|
||||
initContainers:
|
||||
- name: wait-for-db
|
||||
image: mariadb:11.4
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until mariadb-admin ping -h ejbca-db --silent; do
|
||||
echo "waiting for ejbca-db..."; sleep 3
|
||||
done
|
||||
containers:
|
||||
- name: ejbca
|
||||
# Pin an explicit tag. `latest` on a CA is how you get an unplanned
|
||||
# schema migration during an incident.
|
||||
image: keyfactor/ejbca-ce:9.1.1
|
||||
env:
|
||||
- name: DATABASE_JDBC_URL
|
||||
value: jdbc:mariadb://ejbca-db:3306/ejbca?characterEncoding=UTF-8
|
||||
- name: DATABASE_USER
|
||||
valueFrom:
|
||||
secretKeyRef: { name: ejbca-db, key: MARIADB_USER }
|
||||
- name: DATABASE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef: { name: ejbca-db, key: MARIADB_PASSWORD }
|
||||
# Lets EJBCA generate its own server TLS keypair on first boot. The
|
||||
# REST/admin listener is cluster-internal and authenticated by
|
||||
# CLIENT certificate, so a self-signed server cert here is fine —
|
||||
# our enrolment route pins the CA chain explicitly rather than
|
||||
# trusting the public roots. Do not "fix" this with cert-manager
|
||||
# without also updating that pin.
|
||||
- name: TLS_SETUP_ENABLED
|
||||
value: "simple"
|
||||
- name: LOG_LEVEL_APP
|
||||
value: INFO
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
- name: https
|
||||
containerPort: 8443
|
||||
# First boot builds the schema and can take minutes. A tight
|
||||
# startupProbe budget here will CrashLoop a CA that is merely slow.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /ejbca/publicweb/healthcheck/ejbcahealth
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ejbca/publicweb/healthcheck/ejbcahealth
|
||||
port: 8080
|
||||
periodSeconds: 15
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ejbca/publicweb/healthcheck/ejbcahealth
|
||||
port: 8080
|
||||
periodSeconds: 30
|
||||
failureThreshold: 5
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 2Gi
|
||||
limits:
|
||||
memory: 4Gi
|
||||
@@ -0,0 +1,51 @@
|
||||
# PUBLIC surface of the CA — revocation checking ONLY.
|
||||
#
|
||||
# Two prefixes are routed and nothing else. Not the admin web, not the REST API,
|
||||
# not the public enrolment pages (/ejbca/ra/, /ejbca/enrol/). Anything else at
|
||||
# this host 404s because no rule matches it.
|
||||
#
|
||||
# WHY THIS MUST BE PUBLIC AT ALL: every certificate this CA issues carries the
|
||||
# CRL Distribution Point and OCSP responder URL *inside* it, and those URLs are
|
||||
# fetched by whoever is validating the certificate. For internal-only S/MIME that
|
||||
# could stay private — but the moment a signed message leaves the building, the
|
||||
# recipient's mail client resolves these URLs from the outside. They also become
|
||||
# permanent: certificates already issued keep pointing here for their full year,
|
||||
# so this hostname cannot be changed casually. Fix the hostname before the first
|
||||
# real issuance, not after.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: vnc-ca-public
|
||||
namespace: vnc-ca
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
# Revocation data is public by design and must be cacheable — an OCSP
|
||||
# responder that is slow or down makes every client either hang or
|
||||
# soft-fail open, and soft-fail-open is the same as no revocation at all.
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "20"
|
||||
spec:
|
||||
ingressClassName: public
|
||||
tls:
|
||||
- hosts:
|
||||
- ca.sandbox.vnc.de
|
||||
secretName: vnc-ca-public-tls
|
||||
rules:
|
||||
- host: ca.sandbox.vnc.de
|
||||
http:
|
||||
paths:
|
||||
# CRL download — http://ca.sandbox.vnc.de/ejbca/publicweb/crls/...
|
||||
- path: /ejbca/publicweb/crls
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: ejbca
|
||||
port:
|
||||
number: 8080
|
||||
# OCSP responder — POST target for status queries.
|
||||
- path: /ejbca/publicweb/status/ocsp
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: ejbca
|
||||
port:
|
||||
number: 8080
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
# secret.example.yaml is deliberately NOT listed. Apply your filled-in copy
|
||||
# out-of-band so real passwords never pass through a file in this repo.
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- mariadb.yaml
|
||||
- ejbca.yaml
|
||||
- ingress.yaml
|
||||
- networkpolicy.yaml
|
||||
|
||||
# Order matters on a cold cluster: the namespace and the secret must exist before
|
||||
# the workloads. kustomize sorts by kind and handles the namespace; the secret is
|
||||
# on you. See README.md § Install.
|
||||
@@ -0,0 +1,92 @@
|
||||
# MariaDB for EJBCA.
|
||||
#
|
||||
# WHY A REAL DATABASE AND NOT THE EMBEDDED H2: the EJBCA container can run on an
|
||||
# internal H2 database for a quick look, but H2 is explicitly not supported for
|
||||
# anything you intend to keep. Since this sandbox CA has to be *promotable* to
|
||||
# production (your decision: "sandbox first and upgrade later"), the database is
|
||||
# the one thing that must not need re-platforming later — every certificate ever
|
||||
# issued, every revocation, and the intermediate CA key all live in here.
|
||||
#
|
||||
# THIS PVC IS THE CROWN JEWELS. See README.md § Backup.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ejbca-db-data
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
# microk8s default. Confirm with `kubectl get sc` and match your cluster.
|
||||
storageClassName: microk8s-hostpath
|
||||
resources:
|
||||
requests:
|
||||
storage: 8Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ejbca-db
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: ejbca-db
|
||||
ports:
|
||||
- name: mysql
|
||||
port: 3306
|
||||
targetPort: 3306
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ejbca-db
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
replicas: 1
|
||||
# Never run two replicas against one RWO volume, and never roll a new pod up
|
||||
# while the old one still holds the data directory.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: ejbca-db
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ejbca-db
|
||||
spec:
|
||||
containers:
|
||||
- name: mariadb
|
||||
image: mariadb:11.4
|
||||
args:
|
||||
- --character-set-server=utf8mb4
|
||||
- --collation-server=utf8mb4_unicode_ci
|
||||
# EJBCA is case-sensitive about its own table names.
|
||||
- --lower_case_table_names=0
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: ejbca-db
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/mysql
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["healthcheck.sh", "--connect", "--innodb_initialized"]
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["healthcheck.sh", "--connect"]
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: ejbca-db-data
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: vnc-ca
|
||||
labels:
|
||||
# The CA is deliberately in its own namespace, NOT in `vncmail`. The webmail
|
||||
# pod is internet-facing; the CA signs certificates. A compromise of the
|
||||
# former must not be a compromise of the latter, and namespace-scoped RBAC
|
||||
# plus the NetworkPolicy in networkpolicy.yaml are what enforce that.
|
||||
app.kubernetes.io/name: vnc-ca
|
||||
app.kubernetes.io/part-of: vncmail-plus
|
||||
@@ -0,0 +1,82 @@
|
||||
# Default-deny ingress for the CA namespace, then three narrow allowances.
|
||||
#
|
||||
# Without this, the REST API on 8443 is reachable from every pod in the cluster.
|
||||
# It is still client-cert authenticated, so this is defence in depth rather than
|
||||
# the only control — but "the only thing standing between any compromised pod and
|
||||
# a certificate factory is one TLS handshake" is not a position to be in.
|
||||
#
|
||||
# PREREQUISITE: microk8s needs a CNI that enforces NetworkPolicy. The default
|
||||
# (Calico) does. If you are on flannel without a policy plugin these objects
|
||||
# apply cleanly and silently enforce NOTHING — verify with the test in
|
||||
# README.md § Verify the network policy rather than assuming.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-ingress
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Ingress]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-public-web-from-ingress
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: ejbca
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
# Port 8080 (CRL/OCSP) from the ingress controller only.
|
||||
# VERIFY THE NAMESPACE: microk8s' nginx addon has historically used
|
||||
# `ingress`, `kube-system`, and `ingress-nginx` depending on version.
|
||||
# kubectl get pods -A | grep -i ingress
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: ingress
|
||||
ports:
|
||||
- port: 8080
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-rest-from-vncmail
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: ejbca
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
# Port 8443 (REST API) from the webmail namespace only. This is the
|
||||
# enrolment route calling the CA with its RA client certificate.
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: vncmail
|
||||
ports:
|
||||
- port: 8443
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-db-from-ejbca
|
||||
namespace: vnc-ca
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: ejbca-db
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: ejbca
|
||||
ports:
|
||||
- port: 3306
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,35 @@
|
||||
# Template only — DO NOT `kubectl apply` this file and DO NOT commit real values.
|
||||
#
|
||||
# Copy to secret.yaml (gitignored), fill in, apply, then delete your local copy:
|
||||
# cp secret.example.yaml /tmp/ca-secret.yaml
|
||||
# $EDITOR /tmp/ca-secret.yaml
|
||||
# kubectl apply -f /tmp/ca-secret.yaml && shred -u /tmp/ca-secret.yaml
|
||||
#
|
||||
# Generate each password with: openssl rand -base64 24
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ejbca-db
|
||||
namespace: vnc-ca
|
||||
type: Opaque
|
||||
stringData:
|
||||
# MariaDB credentials. The EJBCA database holds the CA private keys (soft
|
||||
# crypto token, encrypted at rest by EJBCA) — treat a dump of it as
|
||||
# equivalent to the intermediate CA key itself.
|
||||
MARIADB_ROOT_PASSWORD: CHANGEME_root
|
||||
MARIADB_USER: ejbca
|
||||
MARIADB_PASSWORD: CHANGEME_ejbca
|
||||
MARIADB_DATABASE: ejbca
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ejbca-app
|
||||
namespace: vnc-ca
|
||||
type: Opaque
|
||||
stringData:
|
||||
# Passphrase protecting EJBCA's internal soft crypto token (the one that
|
||||
# wraps the intermediate CA key). Losing this loses the intermediate.
|
||||
# Back it up somewhere that is NOT this cluster.
|
||||
EJBCA_CRYPTO_TOKEN_PIN: CHANGEME_token
|
||||
@@ -1,34 +0,0 @@
|
||||
# Exposes VNCmail+ at vncmail.sandbox.vnc.de, alongside bulwark.sandbox.vnc.de.
|
||||
# MATCH YOUR CLUSTER — inspect the existing Bulwark ingress and copy its
|
||||
# ingressClassName + TLS/cert-manager annotations:
|
||||
# kubectl get ingress -A | grep bulwark
|
||||
# kubectl get ingress <bulwark-ingress> -n <ns> -o yaml
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
namespace: vncmail
|
||||
annotations:
|
||||
# cert-manager issuer — set to whatever bulwark.sandbox.vnc.de uses.
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
# Mail attachments can be large; raise the nginx body limit.
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "100m"
|
||||
spec:
|
||||
# microk8s ingress addon class is usually "public" (nginx). Confirm with
|
||||
# `kubectl get ingressclass` and match bulwark's.
|
||||
ingressClassName: public
|
||||
tls:
|
||||
- hosts:
|
||||
- vncmail.sandbox.vnc.de
|
||||
secretName: vncmail-plus-tls
|
||||
rules:
|
||||
- host: vncmail.sandbox.vnc.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: vncmail-plus
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,11 @@
|
||||
# Owned by CI (the bump-dev job in .gitlab-ci.yml), not by hand. Kept as its
|
||||
# own Component so CI only ever rewrites this 6-line file, never the parent
|
||||
# overlays/dev/kustomization.yaml (structure/patches there stay under normal
|
||||
# code review — CI regenerating a whole hand-maintained file on every push
|
||||
# would silently revert any change made there between deploys).
|
||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||
kind: Component
|
||||
images:
|
||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
||||
newName: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
||||
newTag: sha-d0a1cee6
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: vncmail
|
||||
resources:
|
||||
- ../../base
|
||||
- namespace.yaml
|
||||
# - secret.yaml # create from secret.example.yaml; not committed
|
||||
|
||||
patches:
|
||||
- path: patch-ingress.yaml
|
||||
- path: patch-image-pull-policy.yaml
|
||||
|
||||
components:
|
||||
- image-tag
|
||||
|
||||
# Targets the dev-k8s-1/2/3 cluster (confirmed via direct access: this is
|
||||
# where ArgoCD already lives). The image tag lives in image-tag/ (a separate
|
||||
# Component CI owns — see .gitlab-ci.yml's bump-dev job) rather than here, so
|
||||
# CI never needs to touch this file.
|
||||
@@ -0,0 +1,29 @@
|
||||
# base/deployment.yaml sets imagePullPolicy: Always, which is the right
|
||||
# default for a mutable tag like :latest. The dev overlay pins an immutable
|
||||
# sha-<commit> tag instead (see image-tag/), and for an immutable tag Always
|
||||
# is pure waste - the content behind that tag can never change, so re-pulling
|
||||
# it on every pod start only adds a registry round-trip and a hard dependency
|
||||
# on the registry being reachable at scheduling time.
|
||||
#
|
||||
# It is also load-bearing right now: until CI can actually push (GitLab's
|
||||
# registry vhost serves Rails, not the registry - see .gitlab-ci.yml's
|
||||
# "Registry history" note), sha- tagged images are side-loaded straight into
|
||||
# each node's containerd:
|
||||
#
|
||||
# docker save --platform linux/amd64 -o vncmail.tar <image>:<tag>
|
||||
# scp vncmail.tar dev-k8s-N:/tmp/ && ssh dev-k8s-N \
|
||||
# 'microk8s ctr images import /tmp/vncmail.tar'
|
||||
#
|
||||
# imported to ALL of dev-k8s-1/2/3 so the pod can schedule anywhere. With
|
||||
# Always, kubelet would ignore that local image and fail on a registry pull
|
||||
# for a tag the registry has never seen.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: vncmail-plus
|
||||
imagePullPolicy: IfNotPresent
|
||||
@@ -0,0 +1,27 @@
|
||||
# dev-k8s cluster confirmed to have a `letsencrypt-staging` ClusterIssuer
|
||||
# already (no `letsencrypt-prod` exists there) - staging avoids burning
|
||||
# Let's Encrypt's real rate limits while this is still being stood up.
|
||||
# vncmail.sandbox.vnc.de DNS does not point here yet either - this is the
|
||||
# intended host, not a live one (see VNCMAIL-SETUP.md for what's still open).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-staging
|
||||
spec:
|
||||
tls:
|
||||
- hosts:
|
||||
- vncmail.sandbox.vnc.de
|
||||
secretName: vncmail-plus-tls
|
||||
rules:
|
||||
- host: vncmail.sandbox.vnc.de
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: vncmail-plus
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,12 @@
|
||||
# Owned by CI (the bump-prod job in .gitlab-ci.yml), not by hand — same
|
||||
# reasoning as overlays/dev/image-tag/. Starts pointed at an obviously-fake
|
||||
# tag on purpose: nothing has been promoted yet, and vncmail-prod's ArgoCD
|
||||
# Application has manual sync anyway, so this being "wrong" doesn't deploy
|
||||
# anything wrong — it just means there's nothing to sync until a real
|
||||
# `git push` to main updates it.
|
||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||
kind: Component
|
||||
images:
|
||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
||||
newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus
|
||||
newTag: not-yet-promoted
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: vncmail-prod
|
||||
resources:
|
||||
- ../../base
|
||||
- namespace.yaml
|
||||
# - secret.yaml # create from secret.example.yaml; not committed
|
||||
|
||||
patches:
|
||||
- path: patch-ingress.yaml
|
||||
- path: patch-deployment.yaml
|
||||
|
||||
components:
|
||||
- image-tag
|
||||
|
||||
# NOT MEANT TO BE SYNCED AS COMMITTED. Scaffolding only (see the pipeline
|
||||
# plan's Phase C/D) — image-tag/'s placeholder tag is obviously-invalid on
|
||||
# purpose. The bump-prod job in .gitlab-ci.yml keeps that tag pointed at
|
||||
# whatever's already on dev once main advances, but vncmail-prod's ArgoCD
|
||||
# Application has manual sync — a human still has to click Sync (or
|
||||
# `argocd app sync vncmail-prod`) for any of this to actually apply.
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: vncmail-prod
|
||||
labels:
|
||||
app.kubernetes.io/part-of: vnclagoon-suite
|
||||
@@ -0,0 +1,9 @@
|
||||
# Basic HA. Still `strategy: Recreate` (inherited from base) since the PVCs
|
||||
# are RWO — 2 replicas doesn't buy zero-downtime rollouts by itself, only
|
||||
# tolerance for a node loss between deploys. Revisit if that's not enough.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
spec:
|
||||
replicas: 2
|
||||
@@ -0,0 +1,31 @@
|
||||
# PLACEHOLDER — the real production hostname has not been decided yet (see
|
||||
# VNCMAIL-SETUP.md / the pipeline plan). vncmail.CHANGEME.invalid is
|
||||
# deliberately unresolvable: applying this overlay as committed will not
|
||||
# issue a cert or route traffic anywhere. Replace both occurrences below,
|
||||
# and the matching TLS secretName, before Phase D (first real prod deploy).
|
||||
#
|
||||
# Targets node1-3 (the HA "prod" cluster). Deliberately does NOT override
|
||||
# base's `cert-manager.io/cluster-issuer: CHANGEME` — node1-3 has ZERO
|
||||
# ClusterIssuers configured today (confirmed via direct access). A human
|
||||
# needs to create a real one there (ACME account, DNS-01 or HTTP-01 solver)
|
||||
# before this can be anything but a placeholder.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: vncmail-plus
|
||||
spec:
|
||||
tls:
|
||||
- hosts:
|
||||
- vncmail.CHANGEME.invalid
|
||||
secretName: vncmail-plus-prod-tls
|
||||
rules:
|
||||
- host: vncmail.CHANGEME.invalid
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: vncmail-plus
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,28 @@
|
||||
# Copy to secret.yaml, fill in real values, and apply. DO NOT commit secret.yaml
|
||||
# (it is gitignored). Generate SESSION_SECRET with: openssl rand -base64 32
|
||||
#
|
||||
# JMAP_SERVER_URL is a PLACEHOLDER — there is no production Stalwart instance
|
||||
# yet. This overlay cannot go live (Phase D) until one exists and this value
|
||||
# points at it for real.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: vncmail-env
|
||||
namespace: vncmail-prod
|
||||
type: Opaque
|
||||
stringData:
|
||||
# Core — connect to Stalwart over JMAP
|
||||
JMAP_SERVER_URL: "https://REPLACE-ME-prod-stalwart-not-yet-deployed.invalid"
|
||||
SESSION_SECRET: "REPLACE_ME__openssl_rand_base64_32"
|
||||
# Branding (theme defaults to VNClagoon in code; these set name + logo)
|
||||
APP_NAME: "VNCmail+"
|
||||
APP_SHORT_NAME: "VNCmail+"
|
||||
LOGIN_COMPANY_NAME: "VNClagoon"
|
||||
LOGIN_LOGO_DARK_URL: "/branding/vncmail-wordmark-on-dark.svg"
|
||||
LOGIN_LOGO_LIGHT_URL: "/branding/vncmail-wordmark-on-light.svg"
|
||||
APP_LOGO_DARK_URL: "/branding/vncmail-wordmark-on-dark.svg"
|
||||
APP_LOGO_LIGHT_URL: "/branding/vncmail-wordmark-on-light.svg"
|
||||
LOGIN_LOGO_MAX_HEIGHT: "52"
|
||||
# Housekeeping
|
||||
BULWARK_UPDATE_CHECK: "off"
|
||||
# Data dirs default to /app/data/* (mounted to the PVCs) — no need to set them.
|
||||
@@ -0,0 +1,113 @@
|
||||
# VNCmail+ — Architecture
|
||||
|
||||
VNCmail+ is VNC's fork of [Bulwark](https://github.com/bulwarkmail/webmail), a
|
||||
Next.js (App Router) webmail client that speaks JMAP to **Stalwart** (the mail
|
||||
server — SMTP/IMAP/JMAP, source of truth for all mail/calendar/contacts/files).
|
||||
VNCmail+ holds no mail data itself; it's a UI + a thin server-side JMAP proxy.
|
||||
|
||||
This doc is the map. For day-to-day sandbox work see
|
||||
[SANDBOX-DEV-MANUAL.md](SANDBOX-DEV-MANUAL.md); for going live at scale see
|
||||
[PRODUCTION-SCALE-OUT-PLAN.md](PRODUCTION-SCALE-OUT-PLAN.md).
|
||||
|
||||
## System diagram
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph Clients
|
||||
Browser["Web browser"]
|
||||
Electron["Electron desktop\n(+ local SQLite/FTS5 search index)"]
|
||||
Mobile["vncmail-native (React Native)\n+ vncmail-relay (push)"]
|
||||
end
|
||||
|
||||
subgraph "dev-k8s-1/2/3 — dev cluster"
|
||||
direction TB
|
||||
TraefikDev["Traefik ingress"]
|
||||
AppDev["VNCmail+ pod(s)\nnamespace: vncmail"]
|
||||
ArgoCD["ArgoCD\n(GitOps controller)"]
|
||||
TraefikDev --> AppDev
|
||||
end
|
||||
|
||||
subgraph "node1/2/3 — prod HA cluster"
|
||||
direction TB
|
||||
TraefikProd["Traefik ingress"]
|
||||
AppProd["VNCmail+ pod(s)\nnamespace: vncmail-prod\n(not live yet)"]
|
||||
Ceph["rook-ceph\n(RWX storage, once wired)"]
|
||||
TraefikProd --> AppProd
|
||||
AppProd -.-> Ceph
|
||||
end
|
||||
|
||||
subgraph "Mail backend (per environment)"
|
||||
Stalwart["Stalwart\nSMTP/IMAP/JMAP server\n(source of truth)"]
|
||||
end
|
||||
|
||||
subgraph "S/MIME internal CA — namespace vnc-ca, isolated"
|
||||
EJBCA["EJBCA\n(cert issuance/enrolment)"]
|
||||
end
|
||||
|
||||
subgraph "GitLab (gitlab.vnc.biz) — canonical repo"
|
||||
MR["MR into dev\n(verify: typecheck/lint/test/build)"]
|
||||
Registry["Container registry\nregistry.gitlab.vnc.biz/.../vncmail-plus"]
|
||||
end
|
||||
|
||||
Browser --> TraefikDev
|
||||
Electron --> TraefikDev
|
||||
Mobile --> TraefikDev
|
||||
Browser -.->|"later, once real"| TraefikProd
|
||||
|
||||
AppDev -->|"JMAP over HTTPS\n(proxy.ts, server-side only)"| Stalwart
|
||||
AppProd -.->|JMAP| Stalwart
|
||||
AppDev -.->|"S/MIME enrolment\n(RA client cert, port 8443)"| EJBCA
|
||||
|
||||
MR -->|merge to dev| Registry
|
||||
Registry -->|"bump-dev job pins the tag"| ArgoCD
|
||||
ArgoCD -->|"sync (auto)"| AppDev
|
||||
Registry -.->|"bump-prod pins the tag\n(no rebuild)"| ArgoCD
|
||||
ArgoCD -.->|"sync — MANUAL, permanent gate"| AppProd
|
||||
```
|
||||
|
||||
## Components
|
||||
|
||||
| Component | What it is | Where |
|
||||
|---|---|---|
|
||||
| **VNCmail+** (this repo) | Next.js 16 App Router webmail UI + server-side JMAP proxy (`proxy.ts`, `app/api/*`). Stateful: writes settings/admin/telemetry to `/app/data/*` — see storage note below. | Container, `vncmail` (dev) / `vncmail-prod` (prod, not live) namespaces |
|
||||
| **Stalwart** | External JMAP/SMTP/IMAP mail server. Owns all mail/calendar/contact/file data. VNCmail+ never touches a database directly — every read/write goes over JMAP. | `stalwart.sandbox.vnc.de` (dev; prod instance doesn't exist yet) |
|
||||
| **EJBCA** (`deploy/k8s/ca/`) | Internal CA issuing S/MIME certs for the S/MIME plugin. Deliberately isolated: own namespace `vnc-ca`, own MariaDB, `NetworkPolicy` allows only the `vncmail` namespace to call its REST API. Root-key ceremony is a manual, human-only runbook — never automated. | `vnc-ca` namespace |
|
||||
| **Electron desktop client** | Same Next.js app, packaged with `electron-builder`, standalone server spawned as a child process. Adds a local encrypted SQLite/FTS5 search index (`lib/mail-index/`) — event-driven, refreshed off the same JMAP push connection, for AI/RAG-style "search your mail" queries. Unsigned builds today (no Apple/Windows code-signing cert yet). | Desktop, not cluster-hosted |
|
||||
| **vncmail-native** (separate repo) | React Native/Expo mobile app, forked from upstream `bulwarkmail/native`. Full JMAP delta-sync engine + local mail replica (bodies, not just the search excerpt Electron keeps) + an FTS5 index over it. **The replica is UNENCRYPTED today** — `STORE_FORMAT = 'sqlite-plain'`, no SQLCipher dependency exists; encryption is a documented future native-build flip, not a shipped property. Do not describe this as encrypted. | Mobile (Android emulator; iOS pending) |
|
||||
| **vncmail-relay** (separate repo) | Push notification relay for the mobile app (forked from `bulwarkmail/relay`). | — |
|
||||
| **GitLab CI** (`.gitlab-ci.yml`) | Builds+pushes container images, bumps a git-tracked image tag. **Never touches any cluster** — no cluster credentials in CI at all. | Runs on a GitLab Runner |
|
||||
| **ArgoCD** | GitOps controller, already installed on `dev-k8s` (found idle with zero Applications when this pipeline was built — more idiomatic than having CI run `kubectl` directly). Watches this repo, applies `deploy/k8s/overlays/{dev,prod}`. `vncmail-dev` = automated sync (once bootstrapped); `vncmail-prod` = **permanently manual sync** — that's the Vercel-style "promote to production" gate. | `argocd` namespace on `dev-k8s`; UI at `https://argo.devcluster.vnc.de` |
|
||||
|
||||
## The two clusters
|
||||
|
||||
| | `dev-k8s-1/2/3` | `node1/node2/node3` |
|
||||
|---|---|---|
|
||||
| Role | dev / sandbox | production (HA) |
|
||||
| Storage | `microk8s-hostpath` only (node-local, single-replica-only) | `rook-ceph`: `ceph-rbd` (RWO, default) **and `ceph-cephfs` (RWX, distributed)** |
|
||||
| Ingress | Traefik | Traefik |
|
||||
| cert-manager issuer | `letsencrypt-staging` | **none configured yet** |
|
||||
| ArgoCD | yes, installed | no — not registered as an ArgoCD-managed cluster yet |
|
||||
| Live workloads today | none (fresh) | none (fresh) |
|
||||
|
||||
Both were confirmed empty when this was written — no `vncmail`, `vnc-ca`, or
|
||||
`stalwart` anything on either cluster. Any reference elsewhere in this repo's
|
||||
history to a "live sandbox at vncmail.sandbox.vnc.de" was aspirational
|
||||
(manifests + docs existed, nothing was ever actually applied).
|
||||
|
||||
## The storage coupling — the one fact that shapes the scale-out plan
|
||||
|
||||
`base/deployment.yaml` mounts 4 PVCs, all `ReadWriteOnce`, `strategy:
|
||||
Recreate`:
|
||||
|
||||
| Dir | Contents | Write pattern |
|
||||
|---|---|---|
|
||||
| `settings` | Per-user encrypted settings (AES-256-GCM, keyed by `hash(username:serverUrl)`) — `lib/settings-sync.ts` | Read+write, per-user |
|
||||
| `admin` (config) | Operator-authored: `config.json`, `policy.json`, admin password hash, plugins, themes, branding uploads | Write-once-ish — can be mounted `:ro` after initial setup (`ADMIN_CONFIG_READONLY=true`, already a supported mode — `lib/admin/paths.ts`) |
|
||||
| `admin-state` | Runtime mutations: login timestamps, audit log, setup token | Always read-write, low volume |
|
||||
| `telemetry` | Version-check / usage state | Read+write, low volume |
|
||||
|
||||
**This is why the app is single-replica today.** RWO + `Recreate` means one
|
||||
pod, one node, ever. It's not a bug — it's the correct choice for a
|
||||
single-sandbox deployment — but it's the first thing that has to change to
|
||||
run more than one replica, which is why it's the opening move in
|
||||
[PRODUCTION-SCALE-OUT-PLAN.md](PRODUCTION-SCALE-OUT-PLAN.md).
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,287 @@
|
||||
> # ⚠️ UPDATE — a replica was later built, and this review is why it is shaped the way it is
|
||||
>
|
||||
> The table below says most of these findings "stopped existing" because the scope change removed
|
||||
> the thing they were about. A replica has since been built (`lib/offline-replica/**`), so that
|
||||
> reasoning was re-examined finding by finding rather than inherited:
|
||||
>
|
||||
> - **C1** — still FIXED, and untouched: the replica adds no new dependency and reuses the guarded
|
||||
> optional require. Both `docker build`s are unaffected.
|
||||
> - **C2, C3, C4, H1, H4** — still MOOT, and moot *for the same reasons*, because the persistent
|
||||
> background worker, the shared registry and the server-side-engine-reads-renderer-state shapes
|
||||
> were **not** reinstated. A cycle is request-scoped work in an API route with no resident
|
||||
> credential; there is no registry and no epoch; one request syncs one account. Had the worker
|
||||
> come back, all five would have come back with it.
|
||||
> - **H2** — still FIXED: the key crosses on an inherited file descriptor, never via environment,
|
||||
> and is zeroed after each job. The replica reuses that channel rather than inventing a second.
|
||||
> - **H3 — BACK IN SCOPE, and the only one that is.** This review was right that the webmail does
|
||||
> local delta arithmetic on mailbox unread counts, and a read-only offline cache underneath it
|
||||
> needs a coherence story. The answer is an ordering rule: the replica is consulted **only after
|
||||
> a read has failed at the transport level**, so it is never a cache in front of the server and
|
||||
> the arithmetic never operates on replica numbers. Enforcing that needed a real signal, because
|
||||
> `lib/jmap/client.ts` swallows read errors and returns plausible success — hence
|
||||
> `lib/jmap/transport-health.ts` and the two-part gate in `lib/offline-fallback-client.ts`.
|
||||
> - The *medium/low* findings (Linux-only API, the vacuous `cipher_version` check, the two bindings
|
||||
> not being interchangeable) were all already fixed in the shipped index and are inherited.
|
||||
>
|
||||
> Nothing in this review turned out to be wrong on re-reading. Its verdict — that the sync-engine
|
||||
> core transfers and the platform-specific sections were where the danger lay — held exactly.
|
||||
|
||||
> # ⚠️ SUPERSEDED — reviews a design that was not built
|
||||
>
|
||||
> This reviews `ELECTRON-OFFLINE-ENGINE-DESIGN.md`, which was **dropped**. Its findings were the
|
||||
> direct cause: seeing them, the human narrowed the requirement from a full offline mail replica to
|
||||
> *"a SQLite index we can prompt against"*, refreshed on each delivery/change event. What shipped is
|
||||
> `lib/mail-index/**` + `app/api/offline/{reindex,search}` — see that doc's superseded note.
|
||||
>
|
||||
> **This review did its job.** Most of its severe findings were resolved by the scope change
|
||||
> removing the thing they were about, which is the strongest outcome a review can have:
|
||||
>
|
||||
> | Finding | Outcome |
|
||||
> |---|---|
|
||||
> | **C1** — `@signalapp/sqlcipher` in `dependencies` breaks both Alpine `docker build`s | **FIXED as specified.** It is an `optionalDependencies` entry with a guarded runtime require (`lib/mail-index/binding.ts`). Both `docker build`s verified passing, and the require verified failing cleanly with MODULE_NOT_FOUND inside the musl image. |
|
||||
> | **C2** — credentials are request-scoped, so no persistent worker can hold them | **MOOT.** There is no worker. Indexing is a normal API route using the request's own `jmap_stalwart_ctx` cookie, via the existing `lib/stalwart/credentials.ts`. |
|
||||
> | **C3** — the OAuth-refresh mitigation is itself the bug | **MOOT, and avoided by construction.** The indexer never touches the refresh-token cookie; it only reads an already-minted auth header, so it cannot rotate a token into a response nobody reads. |
|
||||
> | **C4** — shared `registry.json` breaks the multi-account safety premise | **MOOT.** No registry, no epochs, no concurrent workers. |
|
||||
> | **H1** — a server-side engine can't read a renderer-only setting | **MOOT.** The renderer decides when to index. |
|
||||
> | **H2** — key handoff sequencing, and a nonce via env is readable by same-user processes | **FIXED.** The key crosses on an **inherited file descriptor**, never env, and is fetched per job and zeroed after — not held. Sequencing is moot: the key is fetched when a job runs, not at spawn. |
|
||||
> | **H3** — local unread-count arithmetic needs a coherence story | **MOOT.** A retrieval index does not need to stay coherent with live unread counts. |
|
||||
> | **H4** — no cap on concurrent multi-account sync | **MOOT.** One request, one account. |
|
||||
> | *medium/low:* `getSelectedStorageBackend()` is Linux-only and would crash elsewhere | **FIXED** — platform-guarded. |
|
||||
> | *medium/low:* `cipher_version` check would pass vacuously on zero rows | **FIXED** — the shipped assertion requires a non-empty *string*, and a test reads the raw file bytes for a plaintext canary. |
|
||||
> | *medium/low:* the two bindings are not "the same code either way" | **CONFIRMED true, the hard way.** `@signalapp/sqlcipher` rejects varargs params (`TypeError: Params must be either object or array`) where better-sqlite3 accepts them. Documented in `binding.ts`. |
|
||||
>
|
||||
> Reviewing this file's own accuracy: its two re-executed claims (the binding working in Electron 43,
|
||||
> and `PRAGMA key` being a silent no-op) both held up and both shaped the shipped code.
|
||||
|
||||
# Adversarial review: `docs/ELECTRON-OFFLINE-ENGINE-DESIGN.md`
|
||||
|
||||
Reviewer: independent agent, fresh context, no relation to the design's author. 2026-08-04.
|
||||
|
||||
## Verdict
|
||||
|
||||
**Needs substantial rework before implementation — but narrowly scoped rework.**
|
||||
|
||||
The delta-sync core (everything tagged `[reused]` from M, the mobile design) is genuinely sound
|
||||
and transfers; the reviewer attacked it directly and could not break it. The problem is that **all
|
||||
four genuinely-new sections have an unclosed load-bearing mechanism**, and one of them breaks a
|
||||
build that ships today:
|
||||
|
||||
- §3 (binding choice) contains a packaging decision that breaks the hosted Docker image and the
|
||||
integration fixture.
|
||||
- §2 (process choice) rests on a credential claim that is only true inside an HTTP request.
|
||||
- §6 (key handoff) is under-specified in a way that doesn't work as sequenced, and its "unresolved
|
||||
implementation choice" is not security-neutral.
|
||||
- §5.3/§8.3 (multi-account) breaks the specific premise M's D6 fix relies on.
|
||||
|
||||
Nothing here requires re-architecting the sync engine. Stages B-G can proceed against M as
|
||||
written. Stage A as currently specified would not surface most of this.
|
||||
|
||||
All file:line citations in the design doc that were checked resolve correctly (one trivial
|
||||
miscount, noted at the end) — citation quality is high; the problems are in the reasoning built
|
||||
on top.
|
||||
|
||||
---
|
||||
|
||||
## CRITICAL
|
||||
|
||||
### C1 — Adding `@signalapp/sqlcipher` to `dependencies` breaks the hosted Docker build *and* the integration fixture
|
||||
|
||||
**Where:** §3.3.1 ("entering `dependencies`"), §3.3.5, §2.4, §10.5, E2, §13 item 6.
|
||||
|
||||
`Dockerfile:1-4` — `FROM node:24-alpine`, `RUN npm ci`. `integration/webmail.Dockerfile:12-15` —
|
||||
same, `FROM node:24-alpine` + `npm ci`.
|
||||
|
||||
Verified from the published tarball that `@signalapp/sqlcipher@4.0.3`:
|
||||
- ships **6** prebuilds — `darwin-{arm64,x64}`, `linux-{arm64,x64}`, `win32-{arm64,x64}`. No
|
||||
`linuxmusl-*`. (The design doc's list is exactly right.)
|
||||
- ships **no build sources at all** — published `files` is `dist/*`, `prebuilds`, `README.md`. No
|
||||
`binding.gyp`, no `src/`, no `deps/`.
|
||||
- has `install: node-gyp-build`. `node-gyp-build`'s `bin.js` runs `node-gyp-build-test`; on
|
||||
failure it calls `build()` → spawns `node-gyp rebuild` → `process.exit(code)`.
|
||||
|
||||
No prebuild + no `binding.gyp` ⇒ `node-gyp rebuild` fails ⇒ **`npm ci` exits nonzero**. The Linux
|
||||
prebuild also has a glibc ≥ 2.34 floor, so it could not load on musl even if copied.
|
||||
|
||||
**Concrete failure:** the next `docker build` of the production image fails at line 4.
|
||||
`npm run test:integration` fails to build the webmail container. Neither is gated by
|
||||
`VNCMAIL_DESKTOP_STORE_DIR` — that env var only governs *activation*, not *installation*.
|
||||
|
||||
§3.3.5 dismisses musl as "relevant if an Alpine-based container ever wants the engine — which,
|
||||
per §2.4, it must not" — that reasoning is inverted: the Alpine container doesn't want the engine,
|
||||
it just needs `npm install` to succeed regardless.
|
||||
|
||||
**Fix direction:** `optionalDependencies` + a guarded runtime `require` (which also delivers E2's
|
||||
graceful-load-failure behavior for free), or a separate optional package, or `--omit=optional` in
|
||||
both Dockerfiles. Pick one and say so explicitly; add "`docker build` of both Dockerfiles still
|
||||
succeeds" to the Stage A verification list.
|
||||
|
||||
### C2 — Option A's central justification is only true inside an HTTP request; the Worker credential path does not exist
|
||||
|
||||
**Where:** §2.1-for-1, §1.2, §2.4 (Worker), §5.3, §8.1 triggers T1/T4/T5/T11, §13 item 1.
|
||||
|
||||
Verified in `app/api/auth/session/route.ts` and `app/api/auth/token/route.ts`: every credential
|
||||
read goes through `cookies()` from `next/headers` — request-scoped. `lib/oauth/cookie-config.ts:11`
|
||||
sets `httpOnly: true`. The cookies live in the renderer's cookie jar, not in the server. The
|
||||
standalone server holds no session state whatsoever; it decrypts a cookie per request and
|
||||
discards it.
|
||||
|
||||
So "the credentials are already there... No new credential path, no IPC carrying secrets, no
|
||||
second copy" (§2.1-for-1) is materially overstated. What is actually there is *the ability to
|
||||
decrypt a credential presented on an inbound request* — not a resident credential.
|
||||
|
||||
Consequences the design never addresses:
|
||||
|
||||
1. **T1 ("server process ready + an account's credentials resolvable") cannot fire.** At
|
||||
server-ready there are no cookies anywhere. Nor can T4 (network regained), T5 (`StateChange`
|
||||
on the engine's own socket), or T11 (resume from sleep) — none is an inbound renderer request.
|
||||
2. **A `worker_threads` Worker is a separate execution context with no cookie access at all.**
|
||||
§2.4 mandates the Worker and routes talk to it via `postMessage`, but there's no specified
|
||||
point at which the Worker actually receives credentials.
|
||||
3. The only workable shape is: on the first renderer request, decrypt and hand the **plaintext**
|
||||
credentials to the Worker, which retains them for the process lifetime. That is a new
|
||||
long-lived plaintext secret in a new location — the exact thing §2.1-for-1 claims doesn't
|
||||
happen, and the same category of thing `client.ts:6055-6059` already declined once (a
|
||||
resident credential copy in a process that didn't previously hold one). It also creates an
|
||||
invalidation problem never addressed: password change, logout elsewhere, or a cleared cookie
|
||||
leaves the Worker retrying stale credentials indefinitely (since `AuthenticationError` is
|
||||
correctly never treated as a purge signal) — against a server with failed-auth lockout, this
|
||||
locks the user's account.
|
||||
|
||||
This doesn't kill Option A, but it kills the argument that Option A is free of new secret
|
||||
handling — which was the design's #1 stated reason for choosing it over the alternative. That
|
||||
comparison needs to be redone with the resident-copy cost included, not dropped.
|
||||
|
||||
### C3 — The proposed OAuth-refresh mitigation (E11) is not just unimplementable; it *is* the bug it's meant to prevent
|
||||
|
||||
**Where:** E11 (failure-mode table), §1.2's note about `app/api/auth/token/route.ts:104-106`.
|
||||
|
||||
E11's rule: *"the engine never refreshes independently. It obtains tokens only through the
|
||||
existing `PUT /api/auth/token` route, in-process, so there is exactly one refresher and one
|
||||
rotation writer — the route."*
|
||||
|
||||
But that route: reads the refresh token from the **request's** cookie; writes the rotated token
|
||||
as a `Set-Cookie` on the **response**; and on a 400/401/403 from the identity provider, **deletes**
|
||||
the refresh-token cookie and returns 401.
|
||||
|
||||
An in-process server-side call to that route has no cookie to send (401s immediately), and even
|
||||
if the engine forged one from a resident copy, the rotated token would land in a response the
|
||||
engine discards. Net effect: engine refreshes → identity provider rotates the token → the new
|
||||
token lands in a discarded response → the browser still holds the now-superseded token → the
|
||||
next real refresh from the browser gets rejected → the route deletes the cookie → **the user is
|
||||
silently logged out of that account, and the offline store's credentials are dead.**
|
||||
|
||||
The per-slot lock the design suggests as a fallback does not help — the problem is that cookie
|
||||
state lives in the browser, not that the writes race each other.
|
||||
|
||||
Separately, `PUT /api/auth/session` requires three `sec-fetch-*` headers with a comment claiming
|
||||
"non-browser clients cannot forge these" — a Node-side `fetch` call *can* set all three, silently
|
||||
turning a security control into decoration if any engine path goes through this route. Not
|
||||
discussed in the design at all.
|
||||
|
||||
### C4 — The shared registry file breaks the exact premise the multi-account safety fix relies on
|
||||
|
||||
**Where:** §5.1 (`registry.json`, epoch ownership), §2.4 ("one worker per account"), §4.3 (mutex
|
||||
described as "belt-and-braces"), §7.1 (`completePendingPurges()`), §8.3 cross-account, §5.5.
|
||||
|
||||
The mobile design's cross-account safety guarantee depends explicitly on there being **exactly
|
||||
one writer process-wide** — its own JMAP client is a renderer singleton, so multi-account
|
||||
simultaneous sync was out of scope for it, and its own adversarial review never examined
|
||||
concurrent multi-account execution.
|
||||
|
||||
This design introduces multi-account-simultaneous as "a genuine capability gain" and disposes of
|
||||
the concurrency consequences with a one-line "the jitter matters more here" — but the epoch
|
||||
value (the fencing token the whole safety guarantee rests on) lives in `registry.json`, a single
|
||||
JSON file shared across every account. No SQLite transaction covers a plain JSON file. The
|
||||
argument that a real database transaction demotes the old per-account mutex to
|
||||
"belt-and-braces" is correct for state stored *inside* the SQLite file, and does not apply to
|
||||
`registry.json` at all — which names no owner thread, no lock, and no atomic-write discipline.
|
||||
|
||||
Two concrete failures:
|
||||
1. **Lost epoch bump.** Worker A read-modify-writes the registry to bump account A's epoch
|
||||
(purge, clear, logout). Worker B, holding a stale parse, writes its own update and clobbers
|
||||
A's bump. A's in-flight cycle's next commit now passes the epoch check and lands on top of a
|
||||
wipe — an empty record store with a live, advanced cursor and `resyncRequired: false`, exactly
|
||||
the unreachable-by-design state the mobile design's whole S1 fix exists to prevent.
|
||||
2. **Torn read on a shared file.** Worker B is mid-write; the server's launch-time
|
||||
`completePendingPurges()` reads and the parse throws or yields a partial object. The
|
||||
documented rule ("unreadable → treated as a purge") means a transient concurrency artifact
|
||||
triggers a full purge-and-rebootstrap for accounts that were perfectly fine — and because the
|
||||
file is shared, one torn read can hit every account at once, not just one.
|
||||
|
||||
### Other critical-adjacent findings, condensed
|
||||
|
||||
- **H1** — the "sync enabled" toggle lives in the renderer's local storage; the server-side engine
|
||||
(and its background triggers) has no way to read it, so it will materialize an encrypted store
|
||||
and a keychain entry for accounts that never opted in — precisely the failure the design's own
|
||||
lazy-materialization rule was meant to prevent.
|
||||
- **H2** — the key-handoff sequencing assumes accounts exist at server-spawn time; they don't
|
||||
(accounts are added later, by logging in). The two proposed handoff mechanisms are not
|
||||
equivalent: one of them passes a nonce via the spawned process's environment variables, which
|
||||
are readable by any other process running as the same OS user — defeating the entire point of
|
||||
using the OS keychain in the first place. Needs re-sequencing plus picking the other mechanism
|
||||
on security grounds, not "whichever is cleaner to implement."
|
||||
- **H3** — the "no optimistic-mutation layer exists, so nothing to keep coherent" claim is false;
|
||||
the webmail already does local-delta arithmetic on mailbox unread counts and totals for
|
||||
mark-read/move/delete actions, with a comment referencing a prior production bug from getting
|
||||
this exact kind of cutoff wrong. A read-only offline cache sitting underneath that arithmetic
|
||||
needs an explicit coherence story, which the design currently declares unnecessary.
|
||||
- **H4** — no cap specified on how many accounts sync simultaneously; since this is the same
|
||||
process serving the live webmail UI, an unbounded background sync could contend for the same
|
||||
rate-limited server connection as the user's foreground activity, throttling their visible mail
|
||||
during their own multi-account first sync.
|
||||
- Several medium/low findings: one proposed API call is Linux-only and would crash the app on
|
||||
macOS/Windows if implemented as literally described; the Linux keychain fallback behavior is
|
||||
described slightly wrong (Electron already fails safely there; the real hazard is a *different*
|
||||
API a future maintainer might reach for); the claim that two SQLite bindings are "the same code
|
||||
either way" doesn't hold — verified real API differences exist between them; the "single-user"
|
||||
safety check for the hosted-deployment gate doesn't actually verify what it claims to.
|
||||
|
||||
---
|
||||
|
||||
## What the reviewer independently re-verified (not just re-read)
|
||||
|
||||
Re-ran two of the design's three "verified by execution" claims independently, in Electron 43.2.0
|
||||
itself under the same execution mode the standalone server actually uses:
|
||||
|
||||
1. **`@signalapp/sqlcipher@4.0.3` in Electron 43 — fully re-confirmed by actual re-execution.**
|
||||
Loads with no rebuild, real SQLCipher encryption confirmed (encrypted header, no plaintext
|
||||
canary recoverable from raw bytes, wrong key correctly rejected). The strongest part of the
|
||||
original design.
|
||||
2. **`node:sqlite`'s `PRAGMA key` silent no-op — fully re-confirmed by actual re-execution.** No
|
||||
throrw, mailbox left in cleartext, canary recoverable from raw bytes. The design is right to
|
||||
call this the sharpest landmine found and to mandate a positive verification check after every
|
||||
store open (though the exact check needs a small correction — checking for a non-empty
|
||||
*string* rather than a non-empty *result set*, since the no-cipher case returns zero rows, not
|
||||
an empty string, and a naive string comparison would pass vacuously).
|
||||
3. **The Linux keychain-fallback claim — not independently confirmed, and partially contradicted**
|
||||
by reading Electron's own source and current documentation (no Linux desktop was available to
|
||||
actually execute this one). The decision made (refuse outright rather than risk a false sense
|
||||
of security) stays correct regardless and costs nothing, but the specific mechanism described
|
||||
needs correcting.
|
||||
|
||||
## Recommended gate
|
||||
|
||||
Do not start implementation as currently written. Resolve in this order:
|
||||
1. **C1** — decide the dependency-installation shape so the existing Docker builds keep working;
|
||||
add a Docker-build check to the first implementation step's own verification list.
|
||||
2. **C2 + C3** — specify the credential lifecycle end to end: how a background worker actually
|
||||
gets credentials, where they live, how long, how invalidation reaches them, and how token
|
||||
refresh can work given rotation needs to land in the browser's cookie jar, not a discarded
|
||||
response. This may change the process-architecture verdict; re-run that comparison honestly
|
||||
rather than inheriting the original conclusion.
|
||||
3. **C4** — name a single owner (or a real lock plus atomic write) for the shared registry file,
|
||||
and re-derive the multi-account safety guarantee under concurrent writers rather than citing
|
||||
the mobile design's single-writer proof as if it still applied.
|
||||
4. **H1** — decide where the "sync enabled" setting needs to live (or how the engine learns it)
|
||||
so lazy materialization is actually enforceable from where the engine's triggers fire.
|
||||
5. **H2** — pick the handoff mechanism that doesn't leak via process environment variables, and
|
||||
re-sequence it for accounts that don't exist yet at process-spawn time.
|
||||
6. **H3** — add real coherence rules for the counters/totals the webmail already computes locally,
|
||||
or narrow the offline read path to skip anything those computations touch.
|
||||
7. **H4** — state a concurrency bound and a rule that foreground user activity isn't starved by
|
||||
background multi-account sync.
|
||||
8. The smaller medium/low findings should land in the same pass since they're cheap to fix once
|
||||
noticed.
|
||||
|
||||
Everything reused from the mobile design's core sync-engine logic is safe to build against as
|
||||
written — the problems are entirely in the four sections that are genuinely new to this platform.
|
||||
@@ -0,0 +1,151 @@
|
||||
# Bulwark / VNCmail+ — Offline & Native Client Architecture Analysis
|
||||
|
||||
Date: 2026-08-04 (updated same day — see §7 for a strategy-changing discovery)
|
||||
Scope: what Bulwark (upstream `bulwarkmail/webmail`, forked as VNCmail+) delivers today for
|
||||
offline use, and what has to be built to ship Electron (desktop), Capacitor/React-Native
|
||||
iOS (IPA) and Android (APK) clients with local notifications, an encrypted local search index
|
||||
(SQLite/SQLCipher), and true offline mail.
|
||||
|
||||
> **Note on this file's persistence:** `~/vncmail-plus` is a shared checkout — other sessions
|
||||
> actively commit and switch branches here. An earlier untracked copy of this doc was lost to a
|
||||
> branch switch. Commit this file (or move it somewhere durable) if you want it to survive.
|
||||
|
||||
## 1. Current state of the webmail repo (verified against ~/vncmail-plus source)
|
||||
|
||||
| Area | Status today | Evidence |
|
||||
|---|---|---|
|
||||
| Service worker | Installed, but **caches nothing** — `fetch` handler is a deliberate no-op so the app is never usable offline | `public/sw.js:6-8,36` |
|
||||
| Web app manifest | Present, installable PWA (icons, `protocol_handlers` for mailto/webcal) | `app/manifest.ts` |
|
||||
| Push notifications | **Real** Web Push: VAPID subscribe, `Notification.requestPermission`, SW `push`/`notificationclick` handlers, relayed through an external push relay + a preview API route | `lib/web-push.ts`, `public/sw.js:38-42`, `app/api/push/preview/route.ts` |
|
||||
| Local mail cache | **None.** IndexedDB is used only for plugin/theme blobs; `localStorage` only holds device IDs and Zustand UI-state (`persist()`), never message bodies | `lib/plugin-storage.ts`, `stores/account-store.ts:220` |
|
||||
| Search | Server-side JMAP `Email/query` only, no client index | `lib/jmap/search-utils.ts` |
|
||||
| Local encryption | None for cached data. The one AES-256-GCM routine (`lib/auth/crypto.ts`) encrypts the **session cookie server-side** using `node:crypto` — unusable in a browser/WebView | `lib/auth/crypto.ts` |
|
||||
| Mobile/desktop packaging in *this* repo | **Nothing exists**: no `capacitor.config.ts`, no Electron main/`electron-builder`, no Tauri, no fastlane/gradle/Xcode | confirmed via repo-wide `find`; `.github/workflows/*` |
|
||||
| JMAP client portability | `lib/jmap/client.ts` is pure `fetch()`, no Node-only APIs — portable into a WebView/Electron renderer unchanged | grep for `node:`/`require(` in `lib/jmap/*` = zero hits |
|
||||
|
||||
**Multi-account scope: confirmed YES** — the offline cache must support multiple simultaneous
|
||||
Stalwart accounts per device (matches the webmail's existing `account-registry` store). This
|
||||
multiplies SQLCipher key-management work (§3/§7): one isolated key per account, not one global key.
|
||||
|
||||
## 2. The fork in the road: shell strategy
|
||||
|
||||
**Option A — Native shell over a remote WebView.** Capacitor/Electron just point at the hosted
|
||||
Bulwark URL. Cheapest, ships an APK/IPA/desktop binary fast, gets native push — but is *not*
|
||||
offline.
|
||||
|
||||
**Option B — True offline-first client.** The client authenticates and syncs JMAP data
|
||||
directly, keeps a local encrypted store, and only needs connectivity to *sync*, not to *read*.
|
||||
|
||||
Recommendation stands: **Electron first (Option-B-lite is nearly free there — see §4)**, mobile
|
||||
starts with Option A, then graduates to Option B — **but see §7: for mobile, "graduate to
|
||||
Option B" likely means extending an existing app, not building one from scratch.**
|
||||
|
||||
## 3. Build-vs-buy matrix (webmail-repo-only view — see §7 for the revised mobile view)
|
||||
|
||||
| Component | Off-the-shelf | What you build yourselves |
|
||||
|---|---|---|
|
||||
| Capacitor shell (iOS/Android project scaffolding) | Capacitor CLI generates both native projects | Splash/icons, deep-link config, `capacitor.config.ts` tuning |
|
||||
| Local SQLite | `@capacitor-community/sqlite` — ships **native SQLCipher support** on iOS/Android; web fallback via `jeep-sqlite`/`wa-sqlite` | Schema, JMAP→SQLite mapping, migrations |
|
||||
| SQLCipher key lifecycle | Native Keychain/Keystore APIs (via Capacitor Secure Storage) store the raw key | Key derivation/rotation, **per-account keys** (multi-account confirmed §1), wipe-on-logout |
|
||||
| Full-text search | SQLite FTS5 ships free with SQLite | Tokenizer choice, incremental indexer fed by the sync engine |
|
||||
| Native push | `@capacitor/push-notifications` wraps FCM/APNs | Relay extension, device-token registration, notification-tap deep-linking |
|
||||
| Electron desktop | `electron-builder`; Electron's own cross-platform `Notification` API | Main process booting the existing standalone Next.js server; auto-update wiring |
|
||||
| Background sync | iOS `BGTaskScheduler`, Android `WorkManager` | The actual poll/backoff/delta-fetch job logic |
|
||||
| Biometric app-lock | `capacitor-native-biometric` | UI/UX, fallback-to-passcode flow |
|
||||
| Store release pipeline | Fastlane/EAS-style CI, Apple/Google developer accounts | Signing config, CI secrets, store metadata |
|
||||
|
||||
## 4. Why Electron is the cheap win
|
||||
|
||||
Electron has no server-dependency problem: bundle the standalone Next.js server (the same
|
||||
artifact the `Dockerfile` already produces) inside Electron's Node runtime, open a
|
||||
`BrowserWindow` against `localhost`. Reuses 100% of the existing app including `app/api/**`.
|
||||
Native `Notification` API replaces Web Push entirely on desktop. Ships well before mobile
|
||||
Option B.
|
||||
|
||||
## 5. Phased roadmap
|
||||
|
||||
1. **Fix the service worker** — today's SW intentionally caches nothing (`sw.js:36`). Add
|
||||
Workbox-style precaching of the app shell/static assets. Cheap, immediate PWA-offline-shell
|
||||
improvement, no architecture change.
|
||||
2. **Electron desktop** (§4) — bundle standalone server + BrowserWindow + native Notification +
|
||||
`electron-builder` packaging.
|
||||
3. **Capacitor mobile, Option A (remote shell)** — WebView on the hosted instance, native push
|
||||
registration bridged into the relay, biometric app-lock. Ships an installable APK/IPA fast;
|
||||
not offline yet. **Revisit against §7 before starting — extending `vncmail-native` may replace
|
||||
this step entirely rather than complement it.**
|
||||
4. **JMAP sync engine + SQLite/SQLCipher store** — design delta-sync via
|
||||
`Email/changes`/`Mailbox/changes`; local schema, one key per account (§1); move auth off the
|
||||
Next-only encrypted cookie into secure storage so mobile can talk to Stalwart directly.
|
||||
**§7: `vncmail-native` already has a cruder version of the "local cache" half of this
|
||||
(bulk AsyncStorage download) — the delta-sync/SQLite/SQLCipher/FTS half is still greenfield
|
||||
there too, but auth/JMAP wiring is not.**
|
||||
5. **FTS index + offline compose/outbox** — SQLite FTS5 population job; offline-composed
|
||||
messages queued and replayed via JMAP `Email/set` on reconnect; conflict handling.
|
||||
6. **Platform hardening** — background refresh scheduling, Apple export-compliance declaration
|
||||
(SQLCipher/AES in the binary triggers `ITSAppUsesNonExemptEncryption`), signing/release CI.
|
||||
|
||||
## 6. Open questions — status
|
||||
|
||||
- ~~Does the referenced upstream React Native app already solve native push/device-pairing?~~
|
||||
**RESOLVED — see §7.**
|
||||
- **Is Bulwark upstream planning native clients?** Partially answered by §7: yes, `bulwarkmail/native`
|
||||
is that plan, already public, beta/WIP. Still worth watching its upstream activity before
|
||||
diverging further, since pulling upstream improvements is cheaper than re-diverging an AGPL fork.
|
||||
- **Multi-account scope** — RESOLVED, see §1.
|
||||
|
||||
## 7. 2026-08-04 discovery: an upstream React Native app already exists — re-scope Phase 2
|
||||
|
||||
`bulwarkmail/native` (public, AGPL-3.0-only, Expo SDK 54, beta/WIP) is a React Native mobile
|
||||
client for Bulwark. **Forked to `brvncde-dotcom/vncmail-native`.** It already ships:
|
||||
|
||||
- **Multi-account** JMAP sign-in against any server (e.g. Stalwart).
|
||||
- **QR-code cross-device pairing** — `src/screens/LoginScreen.tsx` + `QrScanModal` +
|
||||
`redeemPairingCode`/OAuth handoff in `src/lib/oauth.ts`. This is the "QR-code SSO login and
|
||||
device pairing" feature referenced in the webmail's `CHANGELOG.md:207`.
|
||||
- **Android push notifications via FCM**, dispatched through a *second* public upstream repo,
|
||||
`bulwarkmail/relay` (also AGPL-3.0) — **this is the actual service behind the webmail's
|
||||
`DEFAULT_RELAY_BASE_URL`**, resolving the open question from the original Phase-2 plan about
|
||||
where that relay's source lives. It terminates JMAP `PushSubscription` pushes and forwards to
|
||||
FCM (mobile) or Web Push (PWA); a single Bulwark-hosted instance serves every opted-in client
|
||||
so self-hosters don't need their own Firebase project — **or you can self-host it** (Docker
|
||||
compose provided) if you want push traffic to never leave VNC infrastructure. **Decided:
|
||||
self-host.** Forked to `brvncde-dotcom/vncmail-relay`. Remaining: dedicated Firebase project
|
||||
for FCM credentials, a VAPID keypair, a microk8s deploy alongside `vncmail-plus` (per
|
||||
`vnclagoon-suite-microfrontends`), and repointing both `vncmail-plus`
|
||||
(`DEFAULT_RELAY_BASE_URL`) and `vncmail-native` at the self-hosted instance. Sequenced in the
|
||||
`VNCprodbuild` skill's Phase 2 step 0.
|
||||
- **A basic offline mail cache already**: `src/lib/offline-sync.ts` (155 lines) bulk-downloads
|
||||
the last N days of mail via `Email/query`+`Email/get` into `src/stores/offline-cache-store.ts`
|
||||
(AsyncStorage-backed, size-capped, evicts oldest), with live progress UI
|
||||
(`OfflineCacheBanner.tsx`). **This is not the delta-sync/SQLite/SQLCipher/FTS engine Phase 2
|
||||
called for** — it's a periodic bulk re-download, not incremental `Email/changes` sync, and
|
||||
storage is plain JSON in AsyncStorage, not an encrypted database — but auth, JMAP wiring, and
|
||||
the UI shell around "offline mail" already exist.
|
||||
- Android release pipeline (`.github/workflows/release-android.yml`, sideload APK from GitHub
|
||||
Releases) and an iOS release pipeline (`release-ios.yml`, `docs/ios-release.md`, TestFlight)
|
||||
**already exist** — iOS *builds*, just without push (see below).
|
||||
|
||||
**Still genuinely missing** (confirmed against its own README + source):
|
||||
- iOS push notifications and client certs — Android-only so far.
|
||||
- No SQLite/SQLCipher/FTS anywhere (`@react-native-async-storage/async-storage` +
|
||||
`expo-secure-store` only) — the encrypted-local-index work is still fully greenfield.
|
||||
**Resolved 2026-08-04:** use `expo-sqlite`'s official `useSQLCipher` config-plugin option
|
||||
(Android/iOS/macOS) rather than a third-party binding. Unusable in Expo Go, so this forces a
|
||||
custom dev client for development going forward — accepted. Stay Continuous-Native-Generation
|
||||
(don't commit `ios`/`android`, let `expo prebuild` regenerate them) rather than going fully
|
||||
bare, since a committed native tree would conflict on every future merge from upstream
|
||||
`bulwarkmail/native`. Detail in the `VNCprodbuild` skill's status log.
|
||||
- Filters & rules, S/MIME, plugins, themes, file storage are UI stubs only.
|
||||
- No Play Store distribution yet.
|
||||
|
||||
**Strategic implication:** for the mobile leg of the native-client roadmap, **extending
|
||||
`vncmail-native` is very likely cheaper than building a Capacitor wrapper around the webmail
|
||||
from scratch** — it already has the parts that were the most speculative/decision-heavy in the
|
||||
original Phase-2 plan (auth, pairing, push wiring, multi-account, a working offline-mail UX
|
||||
shell). The remaining work narrows to: iOS push, replacing the AsyncStorage bulk-cache with a
|
||||
real `Email/changes` delta-sync engine into SQLite/SQLCipher, an FTS5 index, and an
|
||||
offline-compose/outbox queue — i.e., roughly roadmap steps 4–6 above, now scoped against an
|
||||
existing app instead of a blank one. **This should be a formal decision gate before touching
|
||||
Phase 2 further**: adopt `vncmail-native` as the mobile client going forward (dropping/deferring
|
||||
the Capacitor-wraps-webmail plan for mobile), or keep both in parallel. Recommend adopting it —
|
||||
duplicating auth/pairing/push work that already exists and works has no upside.
|
||||
@@ -0,0 +1,168 @@
|
||||
# VNCmail+ — Production Scale-Out Plan (target: 100k+ users, scalable on demand)
|
||||
|
||||
Goal: take VNCmail+ from "doesn't exist on `node1-3`" to a production
|
||||
deployment that can grow past 100k users and be scaled **at any time** —
|
||||
both automatically (load-driven) and on a single manual command (ahead of an
|
||||
expected spike), not just reactively.
|
||||
|
||||
Read [ARCHITECTURE.md](ARCHITECTURE.md) first, especially "The storage
|
||||
coupling" section — it's the reason this is phased the way it is.
|
||||
|
||||
## Where things stand today (verified by direct inspection, not assumed)
|
||||
|
||||
- `node1-3` is a healthy 3-node HA microk8s cluster (rook-ceph, traefik,
|
||||
metallb, cert-manager) with **zero application workloads and zero
|
||||
ClusterIssuers**. It's a clean slate, not a half-finished deployment.
|
||||
- `rook-ceph` is already there, with both `ceph-rbd` (RWO) and **`ceph-cephfs`
|
||||
(RWX, distributed)** StorageClasses available — the key piece that makes
|
||||
multi-replica VNCmail+ possible without inventing new infrastructure.
|
||||
- `cnpg-system` (CloudNativePG, a Postgres operator) is **already installed
|
||||
on both clusters** and currently unused by anything. This is the natural
|
||||
home for the app's mutable state once it moves off local files (Phase 1).
|
||||
- No Prometheus/Grafana/logging stack was found on either cluster — this is
|
||||
a real gap, not a "probably fine," and it's a prerequisite for safe
|
||||
autoscaling (HPA needs a metrics pipeline) and for running anything at
|
||||
100k-user scale with any visibility into it.
|
||||
- Stalwart's own scaling story is **not covered here** — it's a separate
|
||||
system owned by the backend/infra side of this decision. It's called out
|
||||
explicitly at each phase below because VNCmail+ scaling is moot if
|
||||
Stalwart can't handle the same load; plan the two together, not
|
||||
sequentially.
|
||||
|
||||
## Phase 1 — Break the storage coupling (blocking; do this first)
|
||||
|
||||
Today: 4 RWO PVCs, `strategy: Recreate`, one pod max, ever. Two ways to fix,
|
||||
pick based on how much time you have before you need >1 replica:
|
||||
|
||||
**Tactical (fast, days)**: switch the 4 PVCs to the `ceph-cephfs` StorageClass
|
||||
(RWX) and the Deployment `strategy` to `RollingUpdate`. This alone unblocks
|
||||
multiple replicas with no code changes. Real risk: `admin-state`/`telemetry`
|
||||
are multi-writer files on a shared filesystem — fine at low write volume
|
||||
(login timestamps, audit log, version-check state), but it's a shortcut, not
|
||||
the target architecture. `settings` (per-user, keyed by `hash(username:serverUrl)`)
|
||||
has no cross-writer conflict risk since each user only ever writes their own
|
||||
file — this one is safe on RWX indefinitely.
|
||||
|
||||
**Structural (correct, weeks)**: migrate `admin-state` and `telemetry` into
|
||||
CNPG Postgres (already installed, unused) — proper multi-writer semantics,
|
||||
no filesystem-locking edge cases, and it's the natural place for this kind
|
||||
of low-volume operational state anyway. Keep `admin` (config) as a
|
||||
**read-only mount** after setup — `ADMIN_CONFIG_READONLY=true` is already a
|
||||
supported mode (`lib/admin/paths.ts`), so this can be baked into the image
|
||||
or a ConfigMap at deploy time instead of a writable volume at all. `settings`
|
||||
can either stay on CephFS RWX (it's genuinely safe there) or also move to
|
||||
Postgres if you want zero PVCs in the final state.
|
||||
|
||||
Either way: this is the one item that has to happen before Phase 2 means
|
||||
anything. Everything downstream assumes replicas > 1 is possible.
|
||||
|
||||
## Phase 2 — Autoscaling & headroom
|
||||
|
||||
- Install a metrics pipeline (`metrics-server` at minimum for HPA;
|
||||
Prometheus+Grafana for real visibility — see Phase 5, do it once, not twice).
|
||||
- `HorizontalPodAutoscaler` on CPU/memory to start; revisit with a custom
|
||||
metric (JMAP request rate, active WebSocket/SSE connections) once you have
|
||||
real traffic shape.
|
||||
- `PodDisruptionBudget` so rolling updates and node maintenance don't drop
|
||||
below your minimum replica count.
|
||||
- Re-size `resources.requests/limits` from real load-test numbers (Phase 7)
|
||||
— the sandbox's `100m/256Mi` requests are sandbox-appropriate, not
|
||||
production-appropriate; don't carry them forward by default.
|
||||
- **The "scale at any time" requirement**: HPA covers load-driven scaling,
|
||||
but also document (and rehearse once) a single manual command to add
|
||||
capacity ahead of a known event, before HPA would react:
|
||||
`kubectl -n vncmail-prod scale deploy/vncmail-plus --replicas=N` or
|
||||
bumping the HPA's `minReplicas`. This should be a one-line runbook entry,
|
||||
not something someone has to figure out under pressure.
|
||||
|
||||
## Phase 3 — Stalwart scaling (parallel track, not this repo's code)
|
||||
|
||||
VNCmail+ has no database and does no caching of its own — every request is
|
||||
a live JMAP call to Stalwart. At 100k users, Stalwart's own architecture
|
||||
decision matters as much as anything in this repo:
|
||||
|
||||
- Storage backend: Stalwart supports RocksDB (single-node) or FoundationDB
|
||||
(distributed, HA) — FoundationDB is the one that scales past a single
|
||||
node.
|
||||
- Blob storage: point Stalwart's message-blob storage at an S3-compatible
|
||||
backend — rook-ceph's object gateway (RGW), if enabled, is already
|
||||
sitting on the same cluster.
|
||||
- Confirm Stalwart's own capacity plan (connections, IOPS, memory) against
|
||||
the same 100k-user target this doc is aiming for, ideally before Phase 7's
|
||||
load test, not after it fails.
|
||||
|
||||
## Phase 4 — Networking & edge
|
||||
|
||||
- Create a real `ClusterIssuer` on `node1-3` — **none exists today**. Decide
|
||||
ACME account + DNS-01 or HTTP-01 solver before anything else in this phase.
|
||||
- Decide the real production hostname (still an open decision — see
|
||||
`deploy/k8s/overlays/prod/patch-ingress.yaml`'s placeholder).
|
||||
- Rate limiting at the Traefik ingress (a `Middleware` CRD) before opening
|
||||
up publicly at this scale — nothing enforces this today.
|
||||
- Consider a CDN in front of `_next/static` and other cacheable assets to
|
||||
keep origin load down as user count grows.
|
||||
|
||||
## Phase 5 — Observability
|
||||
|
||||
Stand up Prometheus + Grafana (or point at existing org tooling if one
|
||||
already covers this cluster — worth checking before installing a second
|
||||
stack) **before** Phase 2's HPA and **before** Phase 7's load test — you
|
||||
need to see what's happening in both. At minimum: request rate/latency/error
|
||||
rate per pod, JMAP call latency to Stalwart, PVC/CephFS I/O if Phase 1 went
|
||||
the tactical route, and alerting on pod restarts / ImagePullBackOff / cert
|
||||
expiry.
|
||||
|
||||
## Phase 6 — Security hardening
|
||||
|
||||
- `NetworkPolicy` for `vncmail-prod`, mirroring the `vnc-ca` namespace's
|
||||
existing default-deny-plus-narrow-allow pattern — nothing enforces
|
||||
network isolation for `vncmail-prod` today.
|
||||
- Confirm the microk8s CNI on `node1-3` actually enforces `NetworkPolicy`
|
||||
(Calico does, flannel-without-a-policy-plugin silently doesn't — the
|
||||
`vnc-ca` README already flags this exact trap, re-verify for this
|
||||
namespace too rather than assuming).
|
||||
- Image scanning in the CI build stage.
|
||||
- S/MIME CA promotion to prod is its own separate, human-only runbook
|
||||
(`deploy/k8s/ca/README.md` §9) — sequence it, don't bundle it into this
|
||||
plan's steps.
|
||||
|
||||
## Phase 7 — Load testing & capacity planning
|
||||
|
||||
Model the actual target before guessing replica counts: concurrent users,
|
||||
JMAP poll/push connection count, expected sync volume per user, attachment
|
||||
upload size/frequency. Run a load test against a **prod-shaped** deployment
|
||||
(real storage backend from Phase 1, real Stalwart capacity from Phase 3, HPA
|
||||
from Phase 2) before the real cutover — a load test against the sandbox's
|
||||
single-hostPath-replica setup would tell you nothing useful about 100k users.
|
||||
|
||||
Recommend a staged ramp for the actual rollout (soft-launch a cohort →
|
||||
watch Phase 5's dashboards → widen) rather than a single cutover to the full
|
||||
100k target on day one.
|
||||
|
||||
## Phase 8 — Backup & DR
|
||||
|
||||
- rook-ceph snapshot policy for whatever PVCs remain after Phase 1.
|
||||
- Stalwart's own backup strategy (backend-owned, but confirm it exists and
|
||||
is tested — a mail server's data loss is a much worse incident than this
|
||||
app's).
|
||||
- A written, rehearsed restore runbook — not just "backups exist."
|
||||
|
||||
## Go-live sequence (once Phases 1–6 are actually done, not just planned)
|
||||
|
||||
1. Register `node1-3` as an ArgoCD-managed cluster (`argocd cluster add`, or
|
||||
an equivalent ServiceAccount+kubeconfig secret) — not done yet, and
|
||||
deliberately not done before this point.
|
||||
2. Fill in the real values in `deploy/k8s/overlays/prod/` (hostname, prod
|
||||
Stalwart's `JMAP_SERVER_URL`) and apply `deploy/argocd/vncmail-prod-app.yaml`.
|
||||
3. Create the real `vncmail-env` secret + registry pull secret in
|
||||
`vncmail-prod`, by hand, same as dev's one-time bootstrap.
|
||||
4. Merge `dev` → `main` (fast-forward only — `git log dev..main` must be
|
||||
empty first).
|
||||
5. Click **Sync** on `vncmail-prod` in the ArgoCD UI. This stays a
|
||||
permanent manual gate — there is no plan to automate this step, ever.
|
||||
6. Smoke test against the real hostname, watch Phase 5's dashboards, then
|
||||
proceed with Phase 7's staged ramp.
|
||||
|
||||
Nothing in Phases 1–8 requires the go-live sequence to happen first — build
|
||||
and verify the scaling story in isolation (e.g. on `dev-k8s` at smaller
|
||||
scale, or in a throwaway prod-shaped namespace) before the actual cutover.
|
||||
@@ -0,0 +1,131 @@
|
||||
# VNCmail+ — Sandbox / Dev Manual
|
||||
|
||||
Practical, day-to-day guide for developing VNCmail+ and getting changes into
|
||||
the sandbox (`dev-k8s-1/2/3` cluster). For the big picture see
|
||||
[ARCHITECTURE.md](ARCHITECTURE.md); for how to eventually go live see
|
||||
[PRODUCTION-SCALE-OUT-PLAN.md](PRODUCTION-SCALE-OUT-PLAN.md).
|
||||
|
||||
## 1. Repo & branches
|
||||
|
||||
- **Canonical remote**: `gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus`
|
||||
(GitHub `origin` is a passive mirror — never push feature work there).
|
||||
- `main` = production (protected, fast-forward-only from `dev`, no direct pushes).
|
||||
- `dev` = integration/default branch (protected, MR-required).
|
||||
- `vnc/*` or `feature/*` = your working branches → MR into `dev`.
|
||||
|
||||
```bash
|
||||
git clone git@gitlab.vnc.biz:gitlab-instance-b9b5cf2f/vncmail-plus.git
|
||||
cd vncmail-plus
|
||||
git checkout -b vnc/my-change dev
|
||||
```
|
||||
|
||||
## 2. Local development
|
||||
|
||||
```bash
|
||||
npm ci
|
||||
cp .env.dev.example .env.local # built-in mock JMAP server, DEV_MOCK_JMAP=true
|
||||
npm run dev # http://localhost:3000, log in with any username/password
|
||||
```
|
||||
|
||||
The mock JMAP server (`/api/dev-jmap`) means you don't need a real Stalwart
|
||||
instance for UI work. Useful scripts:
|
||||
|
||||
```bash
|
||||
npm run typecheck # tsc --noEmit
|
||||
npm run lint # eslint .
|
||||
npm run test:translations # vitest, fast
|
||||
npm run test:integration # bash integration/run-tests.sh — spins up a REAL
|
||||
# Stalwart via docker-compose (integration/), slower
|
||||
```
|
||||
|
||||
For Electron:
|
||||
|
||||
```bash
|
||||
npm run electron:dev # build:standalone + build:electron + launch
|
||||
npm run test:electron # Playwright, no OS permissions needed (Electron CDP)
|
||||
```
|
||||
|
||||
## 3. Opening a change
|
||||
|
||||
1. Push your branch, open a Merge Request into `dev` on GitLab.
|
||||
2. The `verify` CI job runs automatically: typecheck, lint, unit tests, build.
|
||||
**This is a required check** — it never pushes an image or touches any
|
||||
cluster, just proves the branch builds.
|
||||
3. Get it reviewed, merge.
|
||||
|
||||
## 4. What happens after merge — the pipeline
|
||||
|
||||
```
|
||||
merge to dev
|
||||
→ CI `build`: docker build, push registry.gitlab.vnc.biz/.../vncmail-plus:sha-<sha>
|
||||
→ CI `bump-dev`: commits that tag into
|
||||
deploy/k8s/overlays/dev/image-tag/kustomization.yaml (a small file CI
|
||||
owns — don't hand-edit it, your edit will be overwritten on the next push)
|
||||
→ ArgoCD's `vncmail-dev` Application notices the git change and syncs
|
||||
```
|
||||
|
||||
CI never runs `kubectl` and holds no cluster credentials — it only talks to
|
||||
the registry and to this git repo. ArgoCD (already running on `dev-k8s`,
|
||||
found idle when this pipeline was built) does the actual applying.
|
||||
|
||||
**Until the one-time bootstrap below is done**, `vncmail-dev`'s sync policy
|
||||
is manual on purpose — check its status:
|
||||
|
||||
```bash
|
||||
ssh dev-k8s-1 # or dev-k8s-2 / dev-k8s-3
|
||||
export PATH=/snap/bin:$PATH
|
||||
microk8s kubectl -n argocd get application vncmail-dev
|
||||
```
|
||||
|
||||
Or the UI: `https://argo.devcluster.vnc.de` (`admin` / see
|
||||
`kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d` —
|
||||
rotate after first login).
|
||||
|
||||
## 5. One-time bootstrap (already done or being done — see MR !1 / VNCMAIL-SETUP.md)
|
||||
|
||||
Secrets are **never** managed by CI or ArgoCD — created once, by hand:
|
||||
|
||||
```bash
|
||||
kubectl create secret docker-registry ghcr-pull -n vncmail ... # or make the registry package public
|
||||
cp deploy/k8s/overlays/dev/secret.example.yaml secret.yaml # edit SESSION_SECRET
|
||||
kubectl apply -f secret.yaml
|
||||
```
|
||||
|
||||
Then a first manual Sync in the ArgoCD UI. Once that's clean, flip
|
||||
`deploy/argocd/vncmail-dev-app.yaml`'s `automated:` block on and re-apply —
|
||||
from then on, every merge to `dev` deploys itself.
|
||||
|
||||
## 6. Checking on the running sandbox
|
||||
|
||||
```bash
|
||||
ssh dev-k8s-1
|
||||
export PATH=/snap/bin:$PATH
|
||||
microk8s kubectl -n vncmail get pods,pvc,ingress
|
||||
microk8s kubectl -n vncmail logs deploy/vncmail-plus --tail=100 -f
|
||||
microk8s kubectl -n vncmail rollout status deploy/vncmail-plus
|
||||
```
|
||||
|
||||
No local kubeconfig is assumed — everything above is run over `ssh` directly
|
||||
on a cluster node (`node1/2/3` for prod, `dev-k8s-1/2/3` for dev), using the
|
||||
`microk8s.kubectl` binaries installed there (put `/snap/bin` on `PATH`).
|
||||
|
||||
## 7. Troubleshooting
|
||||
|
||||
| Symptom | Likely cause |
|
||||
|---|---|
|
||||
| ArgoCD shows `vncmail-dev` as `ComparisonError` / SSH handshake failed | The ArgoCD deploy key hasn't been added to GitLab yet (Project → Settings → Repository → Deploy keys) |
|
||||
| `bump-dev`/`bump-prod` CI job fails to push | `CI_JOB_TOKEN` self-push isn't enabled (Settings → CI/CD → Job token permissions), and no `GITLAB_PUSH_TOKEN` variable is set as a fallback |
|
||||
| Pod `ImagePullBackOff` | Registry pull secret missing/expired, or package still private |
|
||||
| Pod `CrashLoopBackOff`, `EACCES` on `/app/data` | `securityContext.fsGroup: 1001` must stay set — some storage drivers also need it on the PVC itself |
|
||||
| Ingress has no address / no cert | Wrong `ingressClassName` (must be `traefik` on both real clusters) or a missing `ClusterIssuer` — `node1-3` (prod) has **none** configured today |
|
||||
| "Ein Fehler ist aufgetreten" on login | Use the full email address (`user@sandbox.vnc.de`), not a bare username — Stalwart auths on the full address |
|
||||
|
||||
## 8. Don't touch (out of scope for day-to-day dev)
|
||||
|
||||
- `deploy/k8s/ca/` (EJBCA internal CA) — separate namespace `vnc-ca`, own
|
||||
README, root-key ceremony is a manual human-only runbook. Never wire CI or
|
||||
ArgoCD automation into it.
|
||||
- `overlays/prod/` and `deploy/argocd/vncmail-prod-app.yaml` — scaffolded,
|
||||
deliberately inert (placeholder hostname, no prod Stalwart, `node1-3` not
|
||||
yet registered with ArgoCD). See [PRODUCTION-SCALE-OUT-PLAN.md](PRODUCTION-SCALE-OUT-PLAN.md)
|
||||
for what has to happen before any of that becomes real.
|
||||
@@ -0,0 +1,311 @@
|
||||
# VNCmail+ Native & Desktop Client — Build Manual
|
||||
|
||||
Status: living document, last updated 2026-08-04. This is the canonical reference for the
|
||||
program that takes VNCmail+ (Bulwark) beyond the hosted webmail: an Electron desktop client, a
|
||||
React Native mobile client, and a self-hosted push relay, working toward true offline mail with
|
||||
an encrypted local index. It consolidates everything decided and built so far across three
|
||||
repositories, so nothing lives only in chat history or a session's memory.
|
||||
|
||||
Companion documents:
|
||||
- `docs/OFFLINE-CLIENT-ARCHITECTURE.md` (in `~/vncmail-plus`) — the original gap analysis this
|
||||
program is based on.
|
||||
- `~/.claude/skills/VNCprodbuild/SKILL.md` — the step-by-step build plan this manual reports
|
||||
progress against. That file is the operational checklist; this file is the narrative reference.
|
||||
|
||||
---
|
||||
|
||||
## 1. Why this program exists
|
||||
|
||||
Bulwark/VNCmail+ is a Next.js JMAP webmail app. As shipped, it has zero offline capability: the
|
||||
service worker caches nothing by design, there's no local mail store, no local search index, and
|
||||
no mobile or desktop native client. The goal of this program is to change that — ship a desktop
|
||||
app, a mobile app, real push notifications, and (eventually) a true offline-first local data
|
||||
layer with an encrypted search index — without re-deriving work that already exists upstream or
|
||||
duplicating effort across repos.
|
||||
|
||||
The single most important strategic fact discovered along the way: **an upstream React Native
|
||||
mobile client already exists and already solves most of what looked like the hardest problems**
|
||||
(auth, multi-account, device pairing, Android push). Building a second mobile client from
|
||||
scratch (e.g. wrapping the webmail in Capacitor) would have thrown that away for no reason. The
|
||||
whole shape of this program reflects that discovery — see §4.
|
||||
|
||||
## 2. Repository map
|
||||
|
||||
All three repos are AGPL-3.0 forks of the upstream Bulwark project (`bulwarkmail` on GitHub),
|
||||
owned by `brvncde-dotcom`:
|
||||
|
||||
| Repo | Forked from | Purpose | Local path |
|
||||
|---|---|---|---|
|
||||
| `vncmail-plus` | `bulwarkmail/webmail` | The Next.js webmail app itself — mail, calendar, contacts, files, admin, plugins. Deploys as a container on microk8s at `vncmail.sandbox.vnc.de`. | `~/vncmail-plus` (⚠️ shared checkout — see §8) |
|
||||
| `vncmail-native` | `bulwarkmail/native` | React Native/Expo mobile client (Android + iOS). Beta/WIP upstream. | `~/vncmail-native` |
|
||||
| `vncmail-relay` | `bulwarkmail/relay` | Push notification relay — terminates JMAP `PushSubscription` pushes, forwards to FCM (mobile) or Web Push (PWA/desktop). Self-hosted per the decision in §4. | `~/vncmail-relay` |
|
||||
|
||||
The webmail's Electron desktop work happens in a **dedicated worktree**, not the shared
|
||||
checkout directly: `~/worktrees/vncmail-electron`, branch `claude/electron-desktop`, based off
|
||||
`vncmail-plus`'s `dev` branch. This will eventually become a PR into `dev`.
|
||||
|
||||
Backend: Stalwart Mail Server (`stalwartlabs/mail-server`), sandbox instance at
|
||||
`stalwart.sandbox.vnc.de`, speaking JMAP (mail/calendar/contacts), SMTP, IMAP, and ManageSieve.
|
||||
|
||||
## 3. Architecture recap
|
||||
|
||||
**The core blocker for "true offline":** `lib/jmap/client.ts` in the webmail is pure `fetch()`,
|
||||
zero Node dependencies — it's portable into any WebView, Electron renderer, or React Native
|
||||
context unchanged. But everything *around* it in the webmail — auth-cookie encryption
|
||||
(`lib/auth/crypto.ts`, uses Node's `node:crypto`), the push relay wiring, and every `app/api/**`
|
||||
route — is server-dependent. A native shell that just points a WebView at a bundled static
|
||||
export of the webmail won't work without either:
|
||||
|
||||
- **Option A — remote shell.** The native wrapper loads the *hosted* URL. Fast, gets native push
|
||||
and an installable binary, but requires connectivity for every screen — not offline.
|
||||
- **Option B — true offline-first.** The client authenticates and syncs JMAP data directly against
|
||||
Stalwart, keeps a local encrypted store, and only needs connectivity to *sync*, not to *read*.
|
||||
|
||||
For **desktop**, this fork-in-the-road barely matters: Electron can bundle the webmail's own
|
||||
standalone Next.js server (the same artifact the `Dockerfile` already produces for the Docker
|
||||
image) inside its Node runtime and point a `BrowserWindow` at `localhost`. That's Option A and B
|
||||
at the same time, practically for free — see §5.
|
||||
|
||||
For **mobile**, the fork-in-the-road is real, which is why §4's discovery mattered so much: it
|
||||
meant Option A was already mostly done upstream, letting the plan skip straight to figuring out
|
||||
what Option B (the real offline engine) needs — instead of re-building Option A from scratch in
|
||||
Capacitor first.
|
||||
|
||||
## 4. Decision log
|
||||
|
||||
Every entry here was an explicit `[DECISION]` gate in the `VNCprodbuild` skill — resolved either
|
||||
by direct research/verification or by explicit user sign-off. Dates are when each was resolved.
|
||||
|
||||
| Date | Decision | Resolution | Why |
|
||||
|---|---|---|---|
|
||||
| 2026-08-04 | Does an offline cache need to support multiple accounts per device? | **Yes** | The webmail already has an `account-registry` store; the mobile offline cache must isolate per-account, including per-account SQLCipher keys later. |
|
||||
| 2026-08-04 | Does an upstream React Native app already solve push/pairing? | **Yes — `bulwarkmail/native`** has multi-account JMAP auth, full QR cross-device pairing, and Android FCM push via `bulwarkmail/relay`. Forked to `vncmail-native`. | Duplicating working auth/pairing/push code in a fresh Capacitor wrapper has no upside. **This flipped the entire mobile strategy** — see §6. |
|
||||
| 2026-08-04 | Self-host the push relay, or depend on upstream's shared hosted instance? | **Self-host.** Forked `bulwarkmail/relay` → `vncmail-relay`. | Keeps push metadata (FCM tokens, timing) on VNC infrastructure rather than a third party's. |
|
||||
| 2026-08-04 | Which SQLite library for the eventual SQLCipher-backed local index, and what Expo workflow? | **`expo-sqlite`'s official `useSQLCipher` config-plugin option** (verified via web search — Android/iOS/macOS support, [docs](https://docs.expo.dev/versions/latest/sdk/sqlite/)), not a third-party binding. **Stay Continuous Native Generation** (don't commit `ios`/`android`, let `expo prebuild` regenerate them) rather than going fully bare. | The official plugin already covers this. SQLCipher is unusable in Expo Go — day-to-day development necessarily moves to a custom dev client, which the user explicitly accepted. Going bare would turn every future merge from upstream `bulwarkmail/native` into a native-project merge conflict, for no offsetting benefit. |
|
||||
| 2026-08-04 | Electron's background/foreground notification strategy: JMAP WebSocket push, polling, or quit-to-tray? | **JMAP WebSocket push, implemented with automatic SSE fallback — see caveat below.** Confirmed live and available: the Stalwart sandbox's JMAP session resource advertises `urn:ietf:params:jmap:websocket` with `supportsPush: true` (`wss://stalwart.sandbox.vnc.de/jmap/ws`), independently confirmed by two separate build agents. | Lower latency, no polling/backoff logic to write, and it's not hypothetical — it's live on the server this build already targets. |
|
||||
| 2026-08-04 | Code-signing: Apple Developer ID? Windows cert? | **Apple: yes** (also unblocks Phase 2's iOS push) — **human must actually enroll**, no agent can create the account or pay the ~$99/yr fee. **Windows: yes, eventually** — but ship unsigned for now during this build phase. | Signing is a purchase/account-creation action, categorically outside what an agent can do. |
|
||||
|
||||
## 5. Phase 1 — Electron desktop client
|
||||
|
||||
**Location:** `~/worktrees/vncmail-electron`, branch `claude/electron-desktop`, 7+ commits ahead
|
||||
of `dev` as of this writing. Not pushed, no PR yet — review the worktree directly first.
|
||||
|
||||
### What exists
|
||||
|
||||
- `electron/main.ts` — boots the exact standalone Next.js server artifact the `Dockerfile`
|
||||
already produces, as a child process on a random localhost port; opens a `BrowserWindow`
|
||||
pointed at it. No parallel server-bundling approach was invented.
|
||||
- `electron/preload.ts` — `contextBridge` exposing `window.vnc.isElectron` and
|
||||
`window.vnc.showNotification(title, options)`, wired to Electron's native `Notification` API in
|
||||
the main process.
|
||||
- `e2e/electron-smoke.spec.ts` + `playwright.electron.config.ts` — the regression gate, using
|
||||
Playwright's `_electron.launch()`. Run via `npm run test:electron`. Verified green, including
|
||||
against a real packaged (`--dir`) build, not just the dev skeleton.
|
||||
- `electron-builder.config.js` + `scripts/assemble-standalone.mjs` + `scripts/build-electron.mjs`
|
||||
— packaging for macOS (`dmg`/`zip`, x64+arm64), Windows (`nsis`), Linux (`AppImage`/`deb`).
|
||||
Currently unsigned.
|
||||
- `electron-updater` wired against GitHub Releases (`brvncde-dotcom/vncmail-plus`), defensively
|
||||
wrapped so a failed update check never crashes the app.
|
||||
- `.github/workflows/electron-build.yml` — CI matrix (mac/win/linux) with `npm run test:electron`
|
||||
as a required gate before packaging/upload.
|
||||
|
||||
### How to build and run it locally
|
||||
|
||||
```bash
|
||||
cd ~/worktrees/vncmail-electron
|
||||
npm install
|
||||
npm run electron:dev # dev loop against the local Next dev server
|
||||
npm run build:standalone # produces the standalone server artifact (same as Docker uses)
|
||||
npm run build:electron # packages via electron-builder (unsigned)
|
||||
npm run test:electron # the smoke-test regression gate
|
||||
```
|
||||
|
||||
### Two real bugs found and fixed while building this (worth knowing about)
|
||||
|
||||
1. **Repo-wide eslint gap.** `vnc/plugins/smime` (an independent sub-package) was missing from
|
||||
the eslint ignore list, so `npm run lint` / the husky pre-commit hook failed for *any* commit
|
||||
touching that path on `dev`, regardless of what changed. Fixed alongside `repos/**`/
|
||||
`examples/**`. **Flag this for whoever reviews the eventual PR** — it's a shared-config fix
|
||||
unrelated to Electron, worth landing on `dev` on its own merits.
|
||||
2. **electron-builder `extraResources` footgun.** electron-builder's resource-copy step
|
||||
unconditionally drops any directory literally named `node_modules` when copying
|
||||
`extraResources` — it was silently stripping the bundled standalone server's dependencies and
|
||||
crashing on launch with `Cannot find module 'next'`. Caught only because the build was
|
||||
actually launched and tested, not just configured. Worth remembering for any future
|
||||
electron-builder work generally, not just this project.
|
||||
|
||||
### Still open
|
||||
|
||||
- **JMAP WebSocket push implementation** (skill steps 6-7) — **DONE.** `getWebSocketUrl()`
|
||||
discovers the endpoint from the session's own capability object (never hardcoded), with
|
||||
exponential-jitter reconnect (200ms base / 5s cap / 3-attempt circuit breaker) and a 30s
|
||||
heartbeat, falling back to the existing SSE/polling chain on failure. A real end-to-end
|
||||
integration test (`integration/tests/11-electron-notification.spec.ts`) logs into the actual
|
||||
Stalwart docker fixture, injects mail over real SMTP, and asserts the native notification
|
||||
fires — not a mocked path. Two real bugs were found and fixed building this: production CSP
|
||||
blocked `wss:` outright (the feature was completely inert in any production build until
|
||||
fixed), and the original backoff timing had a window where a real delivery could be silently
|
||||
missed during a retry cycle.
|
||||
**Caveat, found empirically against the real sandbox server:** `stalwart.sandbox.vnc.de`'s
|
||||
`/jmap/ws` endpoint requires the same HTTP `Authorization` header as every other JMAP endpoint
|
||||
*on the WebSocket handshake itself* — which the browser `WebSocket` API cannot attach (browsers
|
||||
don't allow custom headers on the handshake request). Against this specific server, the client
|
||||
will therefore always fail the WS handshake and fall back to SSE — correctly, by design, but it
|
||||
means "live WebSocket push" is currently unreachable in practice from a browser/Electron
|
||||
client, not just theoretically available. Fixing this for real would need a server-side
|
||||
accommodation (e.g. a short-lived token passed as a WS subprotocol or query parameter) — that's
|
||||
a Stalwart-side change, out of scope for this client work. Functionally nothing is broken (SSE
|
||||
fallback works), but don't expect WS to actually engage against this sandbox until that's
|
||||
addressed.
|
||||
- **Code signing** — blocked on the human actually enrolling in the Apple Developer Program
|
||||
(§4). Once done, wiring the signing identity + notarization into `electron-builder` and CI
|
||||
secrets is a config change, not a rewrite — the current config is structured for it.
|
||||
- **App icon** — using the 512×512 PWA icon as a stand-in.
|
||||
`public/branding/Bulwark_Icon_App.svg` should be rasterized at 1024×1024+ for a proper icon; no
|
||||
SVG rasterization tooling was available in-agent.
|
||||
- **Internal dogfood gate** — a human should install an unsigned build locally and sign off on
|
||||
UX before this goes any further (wider rollout, PR, etc.).
|
||||
|
||||
## 6. Phase 2 — Native mobile client + push relay
|
||||
|
||||
### 6.1 `vncmail-native` — what it already had vs. what this program added
|
||||
|
||||
Upstream `bulwarkmail/native` (forked as-is, no rewrite) already ships:
|
||||
|
||||
- Multi-account JMAP sign-in against any server.
|
||||
- Full QR-code cross-device pairing (`src/screens/LoginScreen.tsx`, `QrScanModal`,
|
||||
`redeemPairingCode`/OAuth handoff in `src/lib/oauth.ts`).
|
||||
- Android push notifications via FCM, dispatched through `bulwarkmail/relay`.
|
||||
- A basic offline mail cache (`src/lib/offline-sync.ts`, `src/stores/offline-cache-store.ts`) —
|
||||
bulk-downloads the last N days of mail via `Email/query`+`Email/get` into AsyncStorage, with a
|
||||
size cap and eviction. **Not** the delta-sync/SQLCipher/FTS engine §7 describes — a periodic
|
||||
bulk re-download, not incremental sync, plain JSON not an encrypted database.
|
||||
- Android + iOS release pipelines already working (`release-android.yml` sideloads an APK from
|
||||
GitHub Releases; `release-ios.yml` + `docs/ios-release.md` ship to TestFlight) — iOS *builds*
|
||||
already work, just without push (Android-only so far per its own README).
|
||||
|
||||
This program's first pass (2026-08-04) added, without touching any of the above:
|
||||
- Verified `npm install`, typecheck, and the existing test suite all pass cleanly (429/430 tests;
|
||||
one pre-existing, unrelated transform failure in `src/stores/__tests__/auth-store.test.ts`, not
|
||||
introduced by this work — worth a look eventually, not urgent).
|
||||
- Confirmed live reachability to `stalwart.sandbox.vnc.de` (HTTP 307 → `/jmap/session`, valid
|
||||
JMAP session JSON returned) — and incidentally re-confirmed the WebSocket push capability from
|
||||
§4/§5.
|
||||
- Added `.github/workflows/android-emulator-smoke.yml` — builds the debug APK, boots a cached
|
||||
AVD via `reactivecircus/android-emulator-runner`, installs, launches the app, fails on process
|
||||
death or a `FATAL EXCEPTION` in logcat within a settle window.
|
||||
|
||||
### How to build and run it locally
|
||||
|
||||
```bash
|
||||
cd ~/vncmail-native
|
||||
npm install
|
||||
npx expo start # Expo Go works fine UNTIL SQLCipher is added (§4) — see note below
|
||||
```
|
||||
|
||||
**Once SQLCipher work starts (§7):** switch to a custom dev client — `npx expo prebuild` +
|
||||
`npx expo run:android` / `npx expo run:ios`, or an EAS development build. Expo Go cannot run an
|
||||
app with `useSQLCipher` enabled. Do not commit the generated `ios`/`android` directories —
|
||||
Continuous Native Generation regenerates them from config plugins on each build (§4).
|
||||
|
||||
### 6.2 `vncmail-relay` — self-hosted push relay
|
||||
|
||||
Forked as-is from `bulwarkmail/relay`. This program added:
|
||||
|
||||
- `deploy/k8s/{namespace,pvc,secret.example,deployment,service,ingress,kustomization}.yaml` +
|
||||
`deploy/k8s/README.md` — mirrors the conventions already used to deploy `vncmail-plus` on
|
||||
microk8s (same namespace, same Recreate-strategy/PVC pattern). **One deliberately unresolved
|
||||
item:** the relay's own Dockerfile creates its runtime user via unpinned `adduser -S` (unlike
|
||||
`vncmail-plus`'s documented uid 1001) — `runAsUser`/`fsGroup` are left unset in the manifest
|
||||
with instructions to verify against the real built image before first deploy, rather than
|
||||
guessing a UID.
|
||||
- `.github/workflows/docker-publish.yml` — publishes to `ghcr.io/brvncde-dotcom/vncmail-relay`,
|
||||
same multi-arch buildx/digest-merge structure as `vncmail-plus`'s own publish workflow.
|
||||
- `SETUP-VNC.md` — documents a generated VAPID keypair (values are in that file only, referenced
|
||||
by name — not the actual secret — in `secret.example.yaml`'s placeholders) and flags what's
|
||||
still human-owned before this can go live: a dedicated Firebase project + its FCM
|
||||
service-account JSON.
|
||||
|
||||
**Not done, and deliberately so:** no `kubectl apply` was run — there is no kubeconfig available
|
||||
in the build environment; deploying is a human-only action. The manifests and a full runbook are
|
||||
ready in `deploy/k8s/README.md`, waiting on:
|
||||
|
||||
1. Create a dedicated Firebase project (not reusing `vncmail-plus`'s or `src-website`'s) and
|
||||
generate its service-account JSON.
|
||||
2. `kubectl apply` the manifests (with real secrets substituted for `secret.example.yaml`'s
|
||||
placeholders) against the microk8s cluster.
|
||||
3. Once the relay is live and reachable (e.g. `vncmail-relay.sandbox.vnc.de`), repoint both
|
||||
`vncmail-plus`'s `DEFAULT_RELAY_BASE_URL` and `vncmail-native`'s equivalent relay base URL
|
||||
(check `src/api/push.ts`/`src/lib/push-notifications.ts`) at it instead of upstream's shared
|
||||
instance. Re-run the webmail's existing Web Push smoke path end-to-end against the new relay
|
||||
before treating it as the default.
|
||||
|
||||
## 7. Remaining roadmap (not yet started)
|
||||
|
||||
In rough order, per the `VNCprodbuild` skill:
|
||||
|
||||
1. **iOS push (`vncmail-native`)** — blocked on the human's Apple Developer Program enrollment
|
||||
(§4/§5). `vncmail-native` already builds for iOS and ships via TestFlight; only push and
|
||||
client certs are missing.
|
||||
2. **JMAP delta-sync engine** — replace `offline-sync.ts`'s bulk AsyncStorage download with a
|
||||
real `Email/changes`/`Mailbox/changes` cursor-based incremental sync. **This is the
|
||||
highest-stakes step in the entire program** — the skill calls for high/xhigh reasoning effort
|
||||
plus an independent, fresh-context agent adversarially reviewing the design before any
|
||||
implementation starts. Not yet begun.
|
||||
3. **SQLCipher local store** — swap AsyncStorage for `expo-sqlite` with `useSQLCipher: true`
|
||||
(§4), one isolated database/key per account (multi-account confirmed required, §4). Needs an
|
||||
explicit, security-sign-off decision on key derivation/lifecycle (from-password vs.
|
||||
device-random-key wrapped by biometric; wipe-on-logout) before implementation — do not let an
|
||||
agent default this silently.
|
||||
4. **FTS5 search index** — SQLite FTS5 population job tied to the sync engine above.
|
||||
5. **Offline compose/outbox** — queue composed messages while offline, replay via JMAP
|
||||
`Email/set` on reconnect, handle conflicts.
|
||||
6. **Platform hardening** — background refresh scheduling (`BGTaskScheduler`/`WorkManager`),
|
||||
Apple export-compliance declaration (`ITSAppUsesNonExemptEncryption`, triggered once SQLCipher
|
||||
ships in the iOS binary — an agent can draft the text, only a human can file it), Google Play
|
||||
Console account/signing key, final store submissions.
|
||||
7. **Fix the webmail's own no-op service worker** — `public/sw.js` intentionally caches nothing
|
||||
today; adding Workbox-style precaching of the app shell is a cheap, independent improvement to
|
||||
the PWA's offline-shell behavior, unrelated to the native-client work above.
|
||||
|
||||
## 8. Known landmines
|
||||
|
||||
- **`~/vncmail-plus` is a shared, actively-used checkout.** Other sessions commit and switch
|
||||
branches there concurrently. An untracked file written directly into that checkout was lost
|
||||
mid-session to a concurrent branch switch — confirmed incident, 2026-08-04. **Any work meant
|
||||
to persist must go into a dedicated worktree (like `~/worktrees/vncmail-electron`) or be
|
||||
committed immediately** — never leave meaningful uncommitted/untracked work sitting in the
|
||||
shared checkout.
|
||||
- **`~/vncmail-native` and `~/vncmail-relay` are fresh clones** (created 2026-08-04) with no
|
||||
confirmed concurrent-session activity yet — lower risk today, but don't assume that stays true
|
||||
as more work lands there.
|
||||
- **electron-builder + `node_modules`** — see §5's bug writeup; a general electron-builder
|
||||
landmine, not specific to this codebase.
|
||||
- **Expo Go + SQLCipher are mutually exclusive** — see §4/§6.1. The moment `useSQLCipher` is
|
||||
enabled, Expo Go can no longer run the app at all; this is a hard platform constraint, not a
|
||||
configuration bug to work around.
|
||||
- **Electron's random localhost port breaks JMAP login against the sandbox Stalwart —
|
||||
deferred, not fixed, 2026-08-04.** User confirmed testing the packaged Electron app directly
|
||||
against `stalwart.sandbox.vnc.de` (not `localhost`) hit a CORS-shaped login failure. Verified
|
||||
server-side: Stalwart's own CORS headers are correctly wildcarded (`Access-Control-Allow-Origin: *`)
|
||||
on every hop including the `.well-known/jmap` → `/jmap/session` redirect — so this is not a
|
||||
Stalwart allow-list problem. Also found, separately: `vncmail.sandbox.vnc.de` (the documented
|
||||
deployed webmail domain) currently does not resolve (NXDOMAIN) — unrelated to this bug but
|
||||
worth knowing regardless. Leading theory, not yet confirmed against real browser devtools:
|
||||
`electron/main.ts` binds the bundled Next.js server via `server.listen(0, ...)` — a random
|
||||
OS-assigned port every launch — producing a different origin on every run; even if that origin
|
||||
were allow-listed once, it wouldn't stay valid. **User explicitly said skip this for now** —
|
||||
Electron packaging/building itself works, this only affects live login against the sandbox.
|
||||
Fix path when revisited: bind Electron's local server to a fixed port instead of `0`.
|
||||
|
||||
## 9. Before merging any of this
|
||||
|
||||
None of the three repos' branches described here have been pushed or opened as a PR. Before
|
||||
that happens:
|
||||
|
||||
- Run the full existing test/lint suites in each repo, not just the new smoke tests added here.
|
||||
- `vncmail-plus` has its own `VERSION`/`CHANGELOG.md` convention (currently `1.7.8`) — a version
|
||||
bump belongs at actual release/merge time, not mid-feature-branch; this manual deliberately
|
||||
did not touch either file.
|
||||
- Cross-check the eslint-ignore fix (§5) lands even if the rest of the Electron work is split out
|
||||
or delayed — it's an independent, valuable fix on its own.
|
||||
@@ -0,0 +1,113 @@
|
||||
import { test, expect, _electron as electron } from '@playwright/test';
|
||||
import type { ElectronApplication, Page } from '@playwright/test';
|
||||
import path from 'node:path';
|
||||
|
||||
// Live-sandbox verification run (not part of the regular regression suite).
|
||||
//
|
||||
// Unlike e2e/electron-smoke.spec.ts (which deliberately uses a fake
|
||||
// JMAP_SERVER_URL just to skip the /setup wizard, and never expects a real
|
||||
// server on the other end), this spec launches the exact same packaged
|
||||
// artifact against the REAL sandbox JMAP backend at
|
||||
// https://stalwart.sandbox.vnc.de and proves:
|
||||
// 1. the login screen renders with no TLS/network errors reaching that host
|
||||
// 2. submitting an obviously-fake, nonexistent test credential produces a
|
||||
// structured "invalid credentials" style response from the real server
|
||||
// (not a network failure) - proving the renderer -> Next API route ->
|
||||
// real JMAP server round trip works end-to-end, without ever using or
|
||||
// guessing a real account's credentials.
|
||||
const projectRoot = path.resolve(__dirname, '..');
|
||||
const SANDBOX_URL = 'https://stalwart.sandbox.vnc.de';
|
||||
|
||||
test.describe('Electron desktop shell - live sandbox connectivity', () => {
|
||||
let electronApp: ElectronApplication;
|
||||
let appWindow: Page;
|
||||
const pageErrors: Error[] = [];
|
||||
const networkFailures: string[] = [];
|
||||
|
||||
test.beforeAll(async () => {
|
||||
electronApp = await electron.launch({
|
||||
args: [projectRoot],
|
||||
env: {
|
||||
...process.env,
|
||||
JMAP_SERVER_URL: SANDBOX_URL,
|
||||
SESSION_SECRET: process.env.SESSION_SECRET || 'live-sandbox-verification-run',
|
||||
NODE_ENV: 'production',
|
||||
},
|
||||
});
|
||||
|
||||
appWindow = await electronApp.firstWindow();
|
||||
appWindow.on('pageerror', (error) => {
|
||||
pageErrors.push(error);
|
||||
});
|
||||
appWindow.on('requestfailed', (request) => {
|
||||
networkFailures.push(`${request.method()} ${request.url()} - ${request.failure()?.errorText}`);
|
||||
});
|
||||
await appWindow.waitForLoadState('domcontentloaded');
|
||||
});
|
||||
|
||||
test.afterAll(async () => {
|
||||
await electronApp?.close();
|
||||
});
|
||||
|
||||
test('renders the real login screen (not SETUP REQUIRED) with no network/TLS errors', async () => {
|
||||
const bodyText = await appWindow.locator('body').innerText();
|
||||
expect(bodyText).not.toContain('SETUP REQUIRED');
|
||||
expect(bodyText).not.toContain('Setup Required');
|
||||
|
||||
const emailInput = appWindow.locator('input[type="text"]').first();
|
||||
const passwordInput = appWindow.locator('input[type="password"]').first();
|
||||
await expect(emailInput).toBeVisible({ timeout: 20000 });
|
||||
await expect(passwordInput).toBeVisible();
|
||||
|
||||
await appWindow.screenshot({
|
||||
path: path.join(projectRoot, 'live-sandbox-login-screen.png'),
|
||||
fullPage: true,
|
||||
});
|
||||
|
||||
expect(pageErrors.map((e) => e.message).join('\n')).not.toMatch(/ERR_CERT|ERR_CONNECTION|ERR_NAME_NOT_RESOLVED|ECONNREFUSED/i);
|
||||
expect(networkFailures.join('\n')).not.toMatch(/ERR_CERT|ERR_CONNECTION|ERR_NAME_NOT_RESOLVED|ECONNREFUSED/i);
|
||||
});
|
||||
|
||||
test('submitting a nonexistent test credential reaches the real JMAP server and returns a structured auth error (no real account used/guessed)', async () => {
|
||||
const emailInput = appWindow.locator('input[type="text"]').first();
|
||||
const passwordInput = appWindow.locator('input[type="password"]').first();
|
||||
|
||||
// Deliberately fake, nonexistent address - not a real account, not a
|
||||
// guess against one. This only proves the pipe to the real server works.
|
||||
await emailInput.fill('electron-live-sandbox-verify-8f2c@invalid-test.example');
|
||||
await passwordInput.fill('not-a-real-password-8f2c');
|
||||
|
||||
const allResponses: { url: string; status: number }[] = [];
|
||||
appWindow.on('response', (res) => {
|
||||
allResponses.push({ url: res.url(), status: res.status() });
|
||||
});
|
||||
|
||||
await appWindow.locator('button[type="submit"]').first().click();
|
||||
|
||||
// The important assertion: the app renders a structured "invalid
|
||||
// credentials" style error sourced from the real JMAP server's rejection
|
||||
// (visible in whatever locale the app negotiated), not a network/TLS
|
||||
// failure. A real connectivity break to stalwart.sandbox.vnc.de would
|
||||
// instead surface as a generic network-error message or a stuck spinner.
|
||||
const errorBanner = appWindow.getByText(/invalid|ungültig|incorrect|falsch|unauthorized/i).first();
|
||||
await expect(errorBanner).toBeVisible({ timeout: 15000 });
|
||||
const errorText = await errorBanner.innerText();
|
||||
console.log('[live-sandbox] login error banner text:', errorText);
|
||||
expect(errorText.length).toBeGreaterThan(0);
|
||||
expect(errorText).not.toMatch(/network error|failed to fetch|ERR_CERT|ERR_CONNECTION|ECONNREFUSED/i);
|
||||
|
||||
await appWindow.screenshot({
|
||||
path: path.join(projectRoot, 'live-sandbox-after-failed-login-attempt.png'),
|
||||
fullPage: true,
|
||||
});
|
||||
|
||||
console.log('[live-sandbox] ALL responses observed after click:', JSON.stringify(allResponses, null, 2));
|
||||
const authResponses = allResponses.filter((r) => r.url.includes('/api/auth/'));
|
||||
if (authResponses.length > 0) {
|
||||
for (const r of authResponses) {
|
||||
expect(r.status).toBeGreaterThanOrEqual(400);
|
||||
expect(r.status).toBeLessThan(500);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import { test, expect, _electron as electron } from '@playwright/test';
|
||||
import type { ElectronApplication, Page } from '@playwright/test';
|
||||
import path from 'node:path';
|
||||
|
||||
// Regression gate for the Electron desktop shell (electron/main.ts +
|
||||
// electron/preload.ts). Launches the real skeleton - the same standalone
|
||||
// Next.js server artifact the Dockerfile produces, booted as a child
|
||||
// process by main.ts, with a real BrowserWindow on top - and asserts the
|
||||
// login screen renders with zero uncaught page errors. Every later step in
|
||||
// the Electron rollout (notification bridge, realtime sync, packaging) must
|
||||
// keep this green; run it before touching anything else.
|
||||
//
|
||||
// Requires `npm run build:standalone && npm run build:electron` to have run
|
||||
// first (see package.json's `electron:dev`/`test:electron` scripts, which
|
||||
// this suite assumes but does not itself trigger, matching how
|
||||
// playwright.config.ts's browser suite assumes `npm run build` for its own
|
||||
// prod-mode runs).
|
||||
const projectRoot = path.resolve(__dirname, '..');
|
||||
|
||||
test.describe('Electron desktop shell', () => {
|
||||
let electronApp: ElectronApplication;
|
||||
// Named `appWindow`, not `window` - the latter would shadow the DOM
|
||||
// global inside every `appWindow.evaluate(() => window...)` callback
|
||||
// below, silently breaking their typing (evaluate() callbacks run in the
|
||||
// browser context, where `window` must resolve to the DOM global).
|
||||
let appWindow: Page;
|
||||
const pageErrors: Error[] = [];
|
||||
|
||||
test.beforeAll(async () => {
|
||||
electronApp = await electron.launch({
|
||||
args: [projectRoot],
|
||||
env: {
|
||||
...process.env,
|
||||
// Bypass the first-run setup wizard (lib/setup/state.ts's
|
||||
// "bootstrap" state, which 302s everything to /setup) without
|
||||
// needing a reachable JMAP server just to prove the login screen
|
||||
// renders - any non-empty JMAP_SERVER_URL is enough to reach
|
||||
// "env-managed" state and serve the normal app shell.
|
||||
JMAP_SERVER_URL: 'https://stalwart.sandbox.vnc.de',
|
||||
SESSION_SECRET: 'electron-smoke-test-not-for-production',
|
||||
NODE_ENV: 'production',
|
||||
},
|
||||
});
|
||||
|
||||
appWindow = await electronApp.firstWindow();
|
||||
appWindow.on('pageerror', (error) => {
|
||||
pageErrors.push(error);
|
||||
});
|
||||
await appWindow.waitForLoadState('domcontentloaded');
|
||||
});
|
||||
|
||||
test.afterAll(async () => {
|
||||
await electronApp?.close();
|
||||
});
|
||||
|
||||
test('boots the standalone server and renders the login screen', async () => {
|
||||
// Same selectors as e2e/login.spec.ts's browser-based check - the
|
||||
// shell should render the identical login form, not a different view.
|
||||
await expect(appWindow.locator('input[type="text"]')).toBeVisible({ timeout: 20000 });
|
||||
await expect(appWindow.locator('input[type="password"]')).toBeVisible();
|
||||
});
|
||||
|
||||
test('exposes the contextBridge API to the renderer', async () => {
|
||||
const isElectron = await appWindow.evaluate(() => window.vnc?.isElectron);
|
||||
expect(isElectron).toBe(true);
|
||||
});
|
||||
|
||||
test('produces zero uncaught page errors', () => {
|
||||
expect(pageErrors).toEqual([]);
|
||||
});
|
||||
|
||||
test('the native notification bridge round-trips through IPC', async () => {
|
||||
// Not asserting a real OS toast appears - that isn't observable in CI
|
||||
// (headless runners/CI accounts routinely have no notification
|
||||
// permission, and Notification.isSupported() can legitimately be
|
||||
// false). What matters is that window.vnc.showNotification (exposed by
|
||||
// electron/preload.ts's contextBridge) actually reaches the main
|
||||
// process's ipcMain.handle("vnc:show-notification", ...) and resolves -
|
||||
// proving the renderer -> preload -> main -> Electron Notification API
|
||||
// plumbing is wired, not just that `window.vnc` exists.
|
||||
const result = await appWindow.evaluate(async () => {
|
||||
return window.vnc?.showNotification('Electron smoke test', {
|
||||
body: 'IPC round-trip check',
|
||||
});
|
||||
});
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(typeof result?.shown).toBe('boolean');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
// electron-builder config for the VNCmail+ (Bulwark) desktop shell.
|
||||
//
|
||||
// Phase 1 of the VNCprodbuild rollout (~/.claude/skills/VNCprodbuild/SKILL.md):
|
||||
// step 1 - base config, no targets (superseded by this file)
|
||||
// step 6 - this file: real packaging targets + branding icon (below)
|
||||
// step 7 - this file's `publish` block + electron/main.ts's
|
||||
// setupAutoUpdater() - electron-updater against GitHub Releases.
|
||||
// step 9 - still open: code signing / notarization (Apple Developer ID,
|
||||
// optional Windows cert) - both are human-owned purchases, not
|
||||
// configured here. Builds below ship UNSIGNED.
|
||||
module.exports = {
|
||||
appId: "de.vnc.vncmailplus",
|
||||
productName: "VNCmail+",
|
||||
copyright: "Copyright © VNC AG",
|
||||
directories: {
|
||||
output: "dist-electron-builds",
|
||||
},
|
||||
// The packaged app (`files` below) is plain esbuild-bundled JS - no native
|
||||
// node modules of its own. The one native dependency anywhere in the repo,
|
||||
// @signalapp/sqlcipher (used by lib/mail-index/**), ships its own prebuilt
|
||||
// .node binaries for every platform/arch and is copied in wholesale by
|
||||
// scripts/assemble-standalone.mjs as part of the extraResources standalone
|
||||
// bundle below - it is never rebuilt by electron-builder.
|
||||
//
|
||||
// Without this, electron-builder's default @electron/rebuild pass scans
|
||||
// the ENTIRE node_modules tree (not just what's actually packaged) for
|
||||
// anything with a native binding and tries to recompile it from source
|
||||
// against Electron's ABI via node-gyp. That caught @parcel/watcher - a
|
||||
// transitive devDependency of some dev tool, never shipped in this app -
|
||||
// and hard-failed the whole packaging step on any machine without a full
|
||||
// Xcode Command Line Tools install (`gyp: No Xcode or CLT version
|
||||
// detected!`), even though nothing that rebuild step touches is part of
|
||||
// the artifact. Verified by execution: builds failed with npmRebuild at
|
||||
// its default (true) and succeeded once set to false.
|
||||
npmRebuild: false,
|
||||
files: ["dist-electron/**/*", "package.json"],
|
||||
extraResources: [
|
||||
{
|
||||
// Same artifact the Dockerfile bakes into the container image (see
|
||||
// Dockerfile + scripts/assemble-standalone.mjs). electron/main.ts
|
||||
// reads it from process.resourcesPath in packaged builds.
|
||||
//
|
||||
// Deliberately `from: ".next"` (not ".next/standalone") + a filter,
|
||||
// not the more obvious `from: ".next/standalone"` alone:
|
||||
// app-builder-lib's copy filter unconditionally drops a directory
|
||||
// literally named "node_modules" sitting at the copy root (see
|
||||
// node_modules/app-builder-lib/out/util/filter.js's
|
||||
// `relative === "node_modules"` check - it assumes extraResources are
|
||||
// hand-authored assets, not a pre-built server with a traced
|
||||
// node_modules of its own). Copying from one level up so
|
||||
// "standalone/node_modules" is never the literal copy root sidesteps
|
||||
// that check, so the standalone server's node_modules actually
|
||||
// survives into the packaged app instead of getting silently
|
||||
// stripped (caught by manually launching a --dir build - the packaged
|
||||
// server crashed with "Cannot find module 'next'").
|
||||
from: ".next",
|
||||
filter: ["standalone/**/*"],
|
||||
to: ".",
|
||||
},
|
||||
],
|
||||
// Dedicated 1024x1024 app icon: the SRC symbol centred on the SRC dark
|
||||
// ground (#09090b), generated from public/branding/SRC_Symbol.png into
|
||||
// build-resources/app-icon.png (NOT build/ - that's electron-builder's own
|
||||
// gitignored output dir; a source asset living inside it would never get
|
||||
// committed, which is exactly the bug this comment is warning about one
|
||||
// paragraph down). 1024 is the size macOS actually wants for the largest
|
||||
// icns representation ("ICON512@2x"), so nothing gets upsampled.
|
||||
//
|
||||
// Deliberately NOT public/icon-512x512.png (what this used to point at):
|
||||
// that file is the *web* PWA manifest icon, so retouching it for the
|
||||
// desktop app silently changes the browser/PWA install icon too. Separate
|
||||
// source, separate concern.
|
||||
//
|
||||
// NOTE for whoever runs this next: electron-builder does NOT auto-detect a
|
||||
// file named `electron-builder.config.js` - its search list is
|
||||
// electron-builder.{yml,yaml,json,json5,js,cjs,mjs,ts}. Packaging must be
|
||||
// invoked with an explicit `--config electron-builder.config.js`, or every
|
||||
// setting in this file is silently ignored and you get stock defaults
|
||||
// (default Electron atom icon, `dist/` output, productName taken from
|
||||
// package.json's `name`). See the `dist:*` scripts in package.json, which
|
||||
// exist so nobody has to remember that.
|
||||
icon: "build-resources/app-icon.png",
|
||||
mac: {
|
||||
target: [
|
||||
{ target: "dmg", arch: ["x64", "arm64"] },
|
||||
{ target: "zip", arch: ["x64", "arm64"] },
|
||||
],
|
||||
category: "public.app-category.productivity",
|
||||
// No Apple Developer ID yet (VNCprodbuild step 9) - ship unsigned/
|
||||
// un-notarized for now. hardenedRuntime is meaningless without signing
|
||||
// but left explicit so it's obvious what step 9 needs to flip on.
|
||||
hardenedRuntime: false,
|
||||
},
|
||||
afterSign: "scripts/after-sign.cjs",
|
||||
win: {
|
||||
target: [{ target: "nsis", arch: ["x64"] }],
|
||||
},
|
||||
nsis: {
|
||||
oneClick: false,
|
||||
allowToChangeInstallationDirectory: true,
|
||||
},
|
||||
linux: {
|
||||
target: [
|
||||
{ target: "AppImage", arch: ["x64"] },
|
||||
{ target: "deb", arch: ["x64"] },
|
||||
],
|
||||
category: "Network;Email;",
|
||||
},
|
||||
// electron-updater feed (see electron/main.ts's setupAutoUpdater()).
|
||||
// GitHub Releases, not a new distribution channel - the skill's
|
||||
// recommendation since this repo is already private and this needs no
|
||||
// extra infrastructure. "Light decision" per VNCprodbuild step 7, not
|
||||
// blocking, but flagged: switching later (e.g. to a self-hosted update
|
||||
// server) would mean revisiting this block and the `provider` electron-
|
||||
// updater talks to.
|
||||
publish: {
|
||||
provider: "github",
|
||||
owner: "brvncde-dotcom",
|
||||
repo: "vncmail-plus",
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,231 @@
|
||||
// Main-process key service for the local search index.
|
||||
//
|
||||
// The index database is SQLCipher-encrypted with a random per-account 32-byte
|
||||
// key. That key is wrapped with Electron's `safeStorage` (OS keychain / DPAPI /
|
||||
// libsecret-or-kwallet) and stored under the store directory. Only the main
|
||||
// process can call `safeStorage`, but the index itself lives in the standalone
|
||||
// Next.js server child process - so the unwrapped key has to cross one process
|
||||
// boundary.
|
||||
//
|
||||
// TRANSPORT: an inherited file descriptor (fd 3), NOT an environment variable.
|
||||
// A nonce or key passed through the spawned process's environment is readable
|
||||
// by any other process running as the same OS user (`ps eww`, /proc/<pid>/environ),
|
||||
// which would defeat the entire point of using the OS keychain. An inherited fd
|
||||
// is not exposed to process listing. libuv creates extra stdio "pipe" entries
|
||||
// as socketpairs, so fd 3 is duplex - verified by execution through Electron's
|
||||
// own spawn before this was built on.
|
||||
//
|
||||
// The server side asks for a key only when a reindex job actually runs and drops
|
||||
// it when the job finishes (see lib/mail-index/key.ts) - there is no long-lived
|
||||
// resident copy anywhere.
|
||||
import { safeStorage } from "electron";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { Readable, Writable } from "node:stream";
|
||||
|
||||
/** Must match lib/mail-index/paths.ts's accountFileToken(). */
|
||||
function accountFileToken(accountId: string): string {
|
||||
return createHash("sha256").update(accountId, "utf8").digest("hex").slice(0, 32);
|
||||
}
|
||||
|
||||
function keyFilePath(storeDir: string, accountId: string): string {
|
||||
return path.join(storeDir, "keys", `${accountFileToken(accountId)}.bin`);
|
||||
}
|
||||
|
||||
export type KeyServiceFailure =
|
||||
/** No OS keyring at all - see the Linux note in checkEncryptionAvailable(). */
|
||||
| "no-secure-storage"
|
||||
/** Reading/writing the wrapped key file failed. */
|
||||
| "key-io-failed"
|
||||
/** The wrapped key exists but safeStorage could not decrypt it. */
|
||||
| "key-unreadable";
|
||||
|
||||
export class KeyServiceError extends Error {
|
||||
code: KeyServiceFailure;
|
||||
constructor(code: KeyServiceFailure, message: string) {
|
||||
super(message);
|
||||
this.name = "KeyServiceError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides whether we are willing to store an encryption key on this system.
|
||||
*
|
||||
* The Linux caveat is the reason this is a function and not a one-liner:
|
||||
* `safeStorage.isEncryptionAvailable()` can return **true** on Linux while the
|
||||
* data is protected by a hardcoded, publicly-known password, with
|
||||
* `getSelectedStorageBackend()` reporting `basic_text`. That is worse than an
|
||||
* honest failure, because it looks like it worked. So a `basic_text` backend is
|
||||
* treated as "no secure storage" and the feature refuses to materialise
|
||||
* anything - the index is a convenience, and silently pretending a mailbox is
|
||||
* encrypted when it is not is not a trade worth making.
|
||||
*
|
||||
* `getSelectedStorageBackend()` is **Linux-only** and throws elsewhere, hence
|
||||
* the platform guard. Both calls also require `app.whenReady()`.
|
||||
*/
|
||||
export function checkEncryptionAvailable(): { ok: true } | { ok: false; reason: string } {
|
||||
if (!safeStorage.isEncryptionAvailable()) {
|
||||
return { ok: false, reason: "The OS reports no secure storage available for encryption keys." };
|
||||
}
|
||||
if (process.platform === "linux") {
|
||||
let backend: string;
|
||||
try {
|
||||
backend = safeStorage.getSelectedStorageBackend();
|
||||
} catch {
|
||||
// Older/newer Electron, or called too early. Be conservative.
|
||||
return { ok: false, reason: "Could not determine the Linux secret-storage backend." };
|
||||
}
|
||||
if (backend === "basic_text" || backend === "unknown") {
|
||||
return {
|
||||
ok: false,
|
||||
reason:
|
||||
`No OS keyring is available (backend: ${backend}). Electron would "encrypt" the key ` +
|
||||
`with a hardcoded password, which provides no real protection, so the encrypted ` +
|
||||
`local index is disabled on this system.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/** Fetches the account's raw index key, creating and wrapping one on first use. */
|
||||
function getOrCreateKey(storeDir: string, accountId: string): Buffer {
|
||||
const availability = checkEncryptionAvailable();
|
||||
if (!availability.ok) {
|
||||
throw new KeyServiceError("no-secure-storage", availability.reason);
|
||||
}
|
||||
|
||||
const file = keyFilePath(storeDir, accountId);
|
||||
|
||||
if (fs.existsSync(file)) {
|
||||
let wrapped: Buffer;
|
||||
try {
|
||||
wrapped = fs.readFileSync(file);
|
||||
} catch (error) {
|
||||
throw new KeyServiceError("key-io-failed", `Could not read the key file: ${String(error)}`);
|
||||
}
|
||||
let hex: string;
|
||||
try {
|
||||
hex = safeStorage.decryptString(wrapped);
|
||||
} catch (error) {
|
||||
// Most likely cause on macOS: the app's code identity changed (unsigned
|
||||
// builds get a fresh ad-hoc signature per build), so the Keychain ACL no
|
||||
// longer matches. Not recoverable and not a user secret - the caller
|
||||
// deletes the database and re-indexes.
|
||||
throw new KeyServiceError(
|
||||
"key-unreadable",
|
||||
`The stored key could not be decrypted (${String(error)}). It must be recreated.`,
|
||||
);
|
||||
}
|
||||
const key = Buffer.from(hex.trim(), "hex");
|
||||
if (key.length === 32) return key;
|
||||
// Corrupt payload: fall through and mint a new one.
|
||||
}
|
||||
|
||||
const key = randomBytes(32);
|
||||
let wrapped: Buffer;
|
||||
try {
|
||||
wrapped = safeStorage.encryptString(key.toString("hex"));
|
||||
} catch (error) {
|
||||
throw new KeyServiceError("no-secure-storage", `Could not wrap the key: ${String(error)}`);
|
||||
}
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
||||
// Write-then-rename so a crash mid-write cannot leave a truncated wrapped
|
||||
// key that would look like "key-unreadable" forever.
|
||||
const tmp = `${file}.tmp-${process.pid}`;
|
||||
fs.writeFileSync(tmp, wrapped, { mode: 0o600 });
|
||||
fs.renameSync(tmp, file);
|
||||
} catch (error) {
|
||||
throw new KeyServiceError("key-io-failed", `Could not persist the key: ${String(error)}`);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
function deleteKey(storeDir: string, accountId: string): void {
|
||||
try {
|
||||
fs.rmSync(keyFilePath(storeDir, accountId), { force: true });
|
||||
} catch {
|
||||
/* best effort - the caller is purging anyway */
|
||||
}
|
||||
}
|
||||
|
||||
interface Request {
|
||||
id?: unknown;
|
||||
op?: unknown;
|
||||
accountId?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serves newline-delimited JSON requests from the standalone server over the
|
||||
* inherited fd. One line in, one line out, no streaming and no state.
|
||||
*/
|
||||
export function attachKeyService(
|
||||
channel: (Readable & Writable) | null | undefined,
|
||||
storeDir: string,
|
||||
): void {
|
||||
if (!channel) {
|
||||
console.error("[electron] key service: no channel on fd 3; the local index will be disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
let buffer = "";
|
||||
const respond = (payload: Record<string, unknown>) => {
|
||||
try {
|
||||
channel.write(`${JSON.stringify(payload)}\n`);
|
||||
} catch (error) {
|
||||
console.error("[electron] key service: failed to write response:", error);
|
||||
}
|
||||
};
|
||||
|
||||
channel.on("data", (chunk: Buffer | string) => {
|
||||
buffer += typeof chunk === "string" ? chunk : chunk.toString("utf8");
|
||||
// Guard against a peer that never sends a newline.
|
||||
if (buffer.length > 64 * 1024) buffer = "";
|
||||
|
||||
let newline: number;
|
||||
while ((newline = buffer.indexOf("\n")) >= 0) {
|
||||
const line = buffer.slice(0, newline);
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (!line.trim()) continue;
|
||||
|
||||
let req: Request;
|
||||
try {
|
||||
req = JSON.parse(line) as Request;
|
||||
} catch {
|
||||
respond({ id: null, ok: false, code: "bad-request", error: "Malformed request" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const id = typeof req.id === "number" ? req.id : null;
|
||||
const accountId = typeof req.accountId === "string" ? req.accountId : "";
|
||||
if (!accountId) {
|
||||
respond({ id, ok: false, code: "bad-request", error: "Missing accountId" });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
if (req.op === "getIndexKey") {
|
||||
const key = getOrCreateKey(storeDir, accountId);
|
||||
respond({ id, ok: true, key: key.toString("hex") });
|
||||
key.fill(0);
|
||||
} else if (req.op === "deleteIndexKey") {
|
||||
deleteKey(storeDir, accountId);
|
||||
respond({ id, ok: true });
|
||||
} else {
|
||||
respond({ id, ok: false, code: "bad-request", error: `Unknown op: ${String(req.op)}` });
|
||||
}
|
||||
} catch (error) {
|
||||
const code = error instanceof KeyServiceError ? error.code : "key-io-failed";
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
respond({ id, ok: false, code, error: message });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
channel.on("error", (error: unknown) => {
|
||||
console.error("[electron] key service channel error:", error);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
// Electron main process for the VNCmail+ (Bulwark) desktop shell.
|
||||
//
|
||||
// Boots the exact same Next.js "standalone" server artifact the Dockerfile
|
||||
// already produces for production (see next.config.ts's `output:
|
||||
// "standalone"` and the Dockerfile's builder stage) as a child process on a
|
||||
// random localhost port, then opens a BrowserWindow pointed at it. This is
|
||||
// deliberately the same server, not a reimplementation - lib/jmap/client.ts
|
||||
// and every app/api/** route behave identically to the web deployment.
|
||||
import { app, BrowserWindow, ipcMain, Notification } from "electron";
|
||||
import { autoUpdater } from "electron-updater";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { createServer } from "node:net";
|
||||
import { get as httpGet } from "node:http";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import type { Duplex } from "node:stream";
|
||||
import { attachKeyService, checkEncryptionAvailable } from "./key-service";
|
||||
|
||||
let serverProcess: ChildProcess | null = null;
|
||||
let mainWindow: BrowserWindow | null = null;
|
||||
|
||||
/**
|
||||
* Root for the encrypted local search index (lib/mail-index/**). Under
|
||||
* `userData`, so it is per-OS-user and removed with the app's data.
|
||||
*
|
||||
* Passing this to the server child process is what ACTIVATES the index: the
|
||||
* routes 404 without it. That matters because the standalone server is the same
|
||||
* artifact the production Dockerfile ships to multi-tenant deployments, where a
|
||||
* server-side index of every user's mail would be badly wrong. One variable
|
||||
* both enables the feature and supplies its path, so the two cannot drift apart.
|
||||
*/
|
||||
function getIndexStoreDir(): string {
|
||||
return path.join(app.getPath("userData"), "offline");
|
||||
}
|
||||
|
||||
/**
|
||||
* Every writable data dir the standalone server uses, redirected under
|
||||
* `userData`.
|
||||
*
|
||||
* WITHOUT this, all four default to `<cwd>/data/*` (see lib/admin/paths.ts,
|
||||
* lib/settings-sync.ts, lib/telemetry/state.ts, lib/version-check/state.ts),
|
||||
* and in a packaged build cwd is `.../VNCmail+.app/Contents/Resources/standalone`
|
||||
* - i.e. the app writes its own runtime state INSIDE its own bundle. Three
|
||||
* separate failure modes, all observed rather than theorised:
|
||||
*
|
||||
* 1. It INVALIDATES THE CODE SIGNATURE. A signed .app seals its Resources;
|
||||
* writing there breaks the seal, so `codesign --verify` starts failing
|
||||
* ("code has no resources but signature indicates they must be present")
|
||||
* and macOS reports the app as *damaged* on a later launch. Verified on
|
||||
* an installed copy in /Applications: signature valid at install time,
|
||||
* exit 1 after the app had run once and written data/admin + data/telemetry.
|
||||
* Deep-signing the bundle at build time (scripts/after-sign.cjs) is
|
||||
* necessary but NOT sufficient on its own - the app immediately breaks
|
||||
* its own signature at runtime unless the writes go elsewhere.
|
||||
* 2. An app update replaces the bundle, silently destroying the user's admin
|
||||
* config, settings and setup state.
|
||||
* 3. It fails outright wherever the bundle isn't user-writable.
|
||||
*
|
||||
* `userData` is the correct home for per-user mutable state on every platform
|
||||
* and is where the search index already lives, so this keeps one convention.
|
||||
*/
|
||||
function getServerDataDirs(): Record<string, string> {
|
||||
const root = app.getPath("userData");
|
||||
return {
|
||||
ADMIN_CONFIG_DIR: path.join(root, "admin"),
|
||||
ADMIN_STATE_DIR: path.join(root, "admin-state"),
|
||||
SETTINGS_DATA_DIR: path.join(root, "settings"),
|
||||
TELEMETRY_DATA_DIR: path.join(root, "telemetry"),
|
||||
VERSION_CHECK_DATA_DIR: path.join(root, "version-check"),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Desktop-shell defaults for a fresh, un-configured install.
|
||||
*
|
||||
* Setting JMAP_SERVER_URL puts the standalone server into "env-managed"
|
||||
* mode (see lib/setup/state.ts's detectSetupState()) - the ONLY thing that
|
||||
* disables the setup wizard short of an operator finishing it by hand. Every
|
||||
* distributable build of this desktop shell up to 2026-08-05 skipped this,
|
||||
* so handing someone the packaged app landed them on "Bulwark Webmail
|
||||
* Setup" asking for a token out of container logs they have no access to -
|
||||
* caught only by actually launching the packaged .app and looking, not by
|
||||
* reading the build log.
|
||||
*
|
||||
* The rest are CONFIG_ENV_MAP entries (lib/admin/types.ts) that only matter
|
||||
* while env-managed - once an admin completes the wizard, config.json wins
|
||||
* for everything except jmapServerUrl itself. allowCustomJmapEndpoint keeps
|
||||
* the server field on the login screen editable, so this is a starting
|
||||
* point for the sandbox, not a hard lock to it.
|
||||
*
|
||||
* `...process.env` in startStandaloneServer() below is spread AFTER this
|
||||
* object, so a real deployment env (the Dockerfile path, or a future
|
||||
* per-install override) still wins over these defaults.
|
||||
*/
|
||||
function getDesktopDefaults(): Record<string, string> {
|
||||
return {
|
||||
JMAP_SERVER_URL: "https://stalwart.sandbox.vnc.de",
|
||||
APP_NAME: "VNCmail+",
|
||||
APP_SHORT_NAME: "VNCmail+",
|
||||
LOGIN_LOGO_LIGHT_URL: "/branding/SRC_Symbol.png",
|
||||
LOGIN_LOGO_DARK_URL: "/branding/SRC_Symbol.png",
|
||||
LOGIN_COMPANY_NAME: "VNC AG",
|
||||
FAVICON_URL: "/branding/SRC_Symbol.png",
|
||||
ALLOW_CUSTOM_JMAP_ENDPOINT: "true",
|
||||
// The login page's subtitle falls back to the login.title i18n string
|
||||
// whenever it differs from appName (app/(main)/[locale]/login/page.tsx)
|
||||
// - a check clearly written for the original Bulwark/"Webmail" pairing,
|
||||
// where they matched. With APP_NAME overridden to "VNCmail+" they no
|
||||
// longer match, so the raw translation ("Webmail") surfaces instead of
|
||||
// anything brand-appropriate. Hiding the subtitle avoids editing a
|
||||
// shared i18n string that every other deployment (incl. Bulwark
|
||||
// default) still uses - the SRC logo + "VNCmail+" heading is enough
|
||||
// context on its own.
|
||||
LOGIN_SHOW_SUBTITLE: "false",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates the standalone server's entrypoint. Packaged builds ship it as an
|
||||
* extraResource (see electron-builder.config.js) because .next/standalone
|
||||
* isn't inside the app.asar; dev runs read it straight out of the repo via
|
||||
* `npm run build:standalone`.
|
||||
*/
|
||||
function getStandaloneServerEntry(): string {
|
||||
if (app.isPackaged) {
|
||||
return path.join(process.resourcesPath, "standalone", "server.js");
|
||||
}
|
||||
return path.join(app.getAppPath(), ".next", "standalone", "server.js");
|
||||
}
|
||||
|
||||
function getFreePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.unref();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
if (address && typeof address === "object") {
|
||||
const { port } = address;
|
||||
server.close(() => resolve(port));
|
||||
} else {
|
||||
server.close(() => reject(new Error("Could not allocate a free localhost port")));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function waitForServerReady(url: string, timeoutMs = 20000): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
return new Promise((resolve, reject) => {
|
||||
const attempt = () => {
|
||||
const req = httpGet(url, (res) => {
|
||||
res.resume();
|
||||
resolve();
|
||||
});
|
||||
req.on("error", () => {
|
||||
if (Date.now() > deadline) {
|
||||
reject(new Error(`Standalone server never became reachable at ${url}`));
|
||||
return;
|
||||
}
|
||||
setTimeout(attempt, 200);
|
||||
});
|
||||
};
|
||||
attempt();
|
||||
});
|
||||
}
|
||||
|
||||
async function startStandaloneServer(): Promise<string> {
|
||||
const serverEntry = getStandaloneServerEntry();
|
||||
if (!fs.existsSync(serverEntry)) {
|
||||
throw new Error(
|
||||
`Standalone Next.js server not found at ${serverEntry}. Run "npm run build:standalone" first.`,
|
||||
);
|
||||
}
|
||||
|
||||
const port = await getFreePort();
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
|
||||
const storeDir = getIndexStoreDir();
|
||||
const encryption = checkEncryptionAvailable();
|
||||
if (!encryption.ok) {
|
||||
// Refuse rather than degrade. On Linux with no keyring, safeStorage
|
||||
// "succeeds" using a hardcoded public password, which would look like an
|
||||
// encrypted mailbox index while providing no protection. Leaving the env
|
||||
// vars unset makes every index route 404, so the app runs normally without
|
||||
// the feature.
|
||||
console.error(`[electron] local search index disabled: ${encryption.reason}`);
|
||||
}
|
||||
|
||||
// Spawn the Electron binary itself as a plain Node process
|
||||
// (ELECTRON_RUN_AS_NODE) instead of depending on a system Node install -
|
||||
// the packaged app can't assume Node exists on the target machine, and
|
||||
// this keeps dev/packaged behavior identical.
|
||||
//
|
||||
// stdio gains a 4th entry: fd 3 is the key channel for the local index (see
|
||||
// electron/key-service.ts). libuv creates extra stdio "pipe" entries as
|
||||
// socketpairs, so it is duplex in both directions - verified by execution
|
||||
// before this was built on. Deliberately NOT an environment variable: env is
|
||||
// readable by any process running as the same OS user, which would defeat
|
||||
// using the OS keychain at all. The fd NUMBER below is not a secret; only
|
||||
// what travels over it is.
|
||||
serverProcess = spawn(process.execPath, [serverEntry], {
|
||||
env: {
|
||||
// First, so any real deployment env (a future per-install override,
|
||||
// or this same binary run somewhere JMAP_SERVER_URL is already set)
|
||||
// wins over these desktop-shell defaults - see getDesktopDefaults().
|
||||
...getDesktopDefaults(),
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: "1",
|
||||
PORT: String(port),
|
||||
HOSTNAME: "127.0.0.1",
|
||||
NODE_ENV: process.env.NODE_ENV || "production",
|
||||
// Keep all mutable state out of the .app bundle - see
|
||||
// getServerDataDirs() for why that matters. Placed after
|
||||
// ...process.env so the desktop shell's paths win over any inherited
|
||||
// value; the same standalone server run outside Electron (the Docker
|
||||
// image) never executes this and keeps its documented env behaviour.
|
||||
...getServerDataDirs(),
|
||||
...(encryption.ok
|
||||
? { VNCMAIL_DESKTOP_STORE_DIR: storeDir, VNCMAIL_DESKTOP_KEY_FD: "3" }
|
||||
: {}),
|
||||
},
|
||||
stdio: encryption.ok
|
||||
? ["inherit", "inherit", "inherit", "pipe"]
|
||||
: "inherit",
|
||||
});
|
||||
|
||||
if (encryption.ok) {
|
||||
attachKeyService(serverProcess.stdio[3] as Duplex | null, storeDir);
|
||||
}
|
||||
|
||||
serverProcess.on("exit", (code, signal) => {
|
||||
if (code !== 0 && code !== null) {
|
||||
console.error(`[electron] standalone server exited early (code=${code}, signal=${signal})`);
|
||||
}
|
||||
serverProcess = null;
|
||||
});
|
||||
|
||||
await waitForServerReady(url);
|
||||
return url;
|
||||
}
|
||||
|
||||
function stopStandaloneServer(): void {
|
||||
if (serverProcess && !serverProcess.killed) {
|
||||
serverProcess.kill();
|
||||
}
|
||||
serverProcess = null;
|
||||
}
|
||||
|
||||
async function createMainWindow(): Promise<void> {
|
||||
// Test-only escape hatch: when set, skip spawning the standalone server
|
||||
// entirely and load this URL instead. Used by
|
||||
// integration/tests/11-electron-notification.spec.ts, which needs a
|
||||
// dev-mode Next.js server (proxy.ts's CSP only widens connect-src to
|
||||
// allow plain-HTTP/ws JMAP in dev - see that file's comments) to reach
|
||||
// the integration fixture's deliberately-plaintext local Stalwart,
|
||||
// exactly the same trade-off integration/webmail.Dockerfile already makes
|
||||
// for the browser-based integration suite. Never set by real users or by
|
||||
// any of the packaging/CI paths - those always go through
|
||||
// startStandaloneServer() below.
|
||||
const url = process.env.ELECTRON_LOAD_URL || (await startStandaloneServer());
|
||||
|
||||
mainWindow = new BrowserWindow({
|
||||
width: 1280,
|
||||
height: 860,
|
||||
webPreferences: {
|
||||
preload: path.join(__dirname, "preload.js"),
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
sandbox: true,
|
||||
},
|
||||
});
|
||||
|
||||
mainWindow.on("closed", () => {
|
||||
mainWindow = null;
|
||||
});
|
||||
|
||||
await mainWindow.loadURL(url);
|
||||
}
|
||||
|
||||
// --- Native notification bridge --------------------------------------------
|
||||
// Called from the preload's `window.vnc.showNotification` (electron/preload.ts),
|
||||
// itself called from lib/electron-bridge.ts's showElectronNotification(),
|
||||
// itself called from app/(main)/[locale]/page.tsx's "new mail arrived"
|
||||
// effect whenever lib/jmap/client.ts's push pipeline (WebSocket, or its SSE/
|
||||
// polling fallback - see that file's circuit breaker) reports a genuine new
|
||||
// message. Electron's own Notification API is the desktop shell's
|
||||
// notification path - it sits alongside, not in place of, the browser/PWA's
|
||||
// service-worker push path (public/sw.js's `push`/`notificationclick`
|
||||
// handlers + lib/web-push.ts).
|
||||
ipcMain.handle(
|
||||
"vnc:show-notification",
|
||||
(_event, title: string, options?: { body?: string; tag?: string }) => {
|
||||
// Test-only observability hook, read via Playwright's
|
||||
// electronApp.evaluate(({ app }) => ...) - see
|
||||
// integration/tests/11-electron-notification.spec.ts. Not gated behind
|
||||
// NODE_ENV: it's an inert counter with no behavioral effect, cheaper
|
||||
// than maintaining a second code path just for tests.
|
||||
const counters = app as unknown as { __notificationCallCount?: number };
|
||||
counters.__notificationCallCount = (counters.__notificationCallCount ?? 0) + 1;
|
||||
|
||||
if (!Notification.isSupported()) {
|
||||
return { shown: false };
|
||||
}
|
||||
const notification = new Notification({
|
||||
title,
|
||||
body: options?.body ?? "",
|
||||
});
|
||||
notification.show();
|
||||
return { shown: true };
|
||||
},
|
||||
);
|
||||
|
||||
// --- Auto-update -------------------------------------------------------
|
||||
// GitHub Releases as the update feed (electron-builder.config.js's
|
||||
// `publish` block) - the skill's recommendation over standing up a new
|
||||
// distribution channel, since the repo is already private. "Light
|
||||
// decision" per VNCprodbuild step 7, not re-litigated here.
|
||||
//
|
||||
// Deliberately best-effort: there's no code signing yet (step 9), so on
|
||||
// macOS in particular an update download/install can fail signature
|
||||
// verification. A failed check must never take the app down - it's
|
||||
// background maintenance, not something the user is blocked on.
|
||||
function setupAutoUpdater(): void {
|
||||
if (!app.isPackaged) {
|
||||
// Unpacked dev/test runs (npm run electron:dev, the Playwright smoke
|
||||
// test) have no latest.yml alongside them - checking would just log a
|
||||
// noisy 404 against GitHub Releases for every dev run.
|
||||
return;
|
||||
}
|
||||
autoUpdater.autoDownload = true;
|
||||
autoUpdater.autoInstallOnAppQuit = true;
|
||||
autoUpdater.on("error", (error) => {
|
||||
console.error("[electron] auto-update error:", error);
|
||||
});
|
||||
autoUpdater.checkForUpdatesAndNotify().catch((error) => {
|
||||
console.error("[electron] checkForUpdatesAndNotify failed:", error);
|
||||
});
|
||||
}
|
||||
|
||||
app.whenReady().then(() => {
|
||||
void createMainWindow();
|
||||
setupAutoUpdater();
|
||||
});
|
||||
|
||||
app.on("window-all-closed", () => {
|
||||
stopStandaloneServer();
|
||||
if (process.platform !== "darwin") {
|
||||
app.quit();
|
||||
}
|
||||
});
|
||||
|
||||
app.on("before-quit", () => {
|
||||
stopStandaloneServer();
|
||||
});
|
||||
|
||||
app.on("activate", () => {
|
||||
if (BrowserWindow.getAllWindows().length === 0) {
|
||||
void createMainWindow();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
// Preload script for the VNCmail+ desktop shell. Runs in an isolated
|
||||
// context with access to Node APIs, and exposes a minimal, explicit surface
|
||||
// to the renderer via contextBridge - the renderer never gets direct Node or
|
||||
// Electron access (contextIsolation + nodeIntegration: false, see main.ts).
|
||||
import { contextBridge, ipcRenderer } from "electron";
|
||||
|
||||
export interface ShowNotificationOptions {
|
||||
body?: string;
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
export interface ShowNotificationResult {
|
||||
shown: boolean;
|
||||
}
|
||||
|
||||
contextBridge.exposeInMainWorld("vnc", {
|
||||
isElectron: true,
|
||||
// Routes to Electron's own Notification API in main.ts (ipcMain.handle
|
||||
// "vnc:show-notification"). This is the desktop shell's native
|
||||
// notification path - it does not replace lib/web-push.ts's Web Push
|
||||
// (VAPID) path, which is what the browser/PWA deployment still uses.
|
||||
showNotification: (
|
||||
title: string,
|
||||
options?: ShowNotificationOptions,
|
||||
): Promise<ShowNotificationResult> =>
|
||||
ipcRenderer.invoke("vnc:show-notification", title, options),
|
||||
});
|
||||
@@ -53,6 +53,18 @@ export default [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Plain Node scripts (electron bundling/packaging helpers) - not React/
|
||||
// browser code, so they get node globals only, no react/jsx parsing.
|
||||
files: ["scripts/**/*.{mjs,cjs,js}"],
|
||||
languageOptions: {
|
||||
ecmaVersion: "latest",
|
||||
sourceType: "module",
|
||||
globals: {
|
||||
...globals.node,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ["**/*.test.{ts,tsx}", "**/*.spec.{ts,tsx}"],
|
||||
languageOptions: {
|
||||
@@ -70,6 +82,8 @@ export default [
|
||||
{
|
||||
ignores: [
|
||||
".next/**",
|
||||
"dist-electron/**",
|
||||
"dist-electron-builds/**",
|
||||
"node_modules/**",
|
||||
"repos/**",
|
||||
"data/admin/plugins/**",
|
||||
@@ -81,6 +95,15 @@ export default [
|
||||
"benchmark/**",
|
||||
"examples/**",
|
||||
"integration/**",
|
||||
// Independent sub-package with its own package.json/build (esbuild,
|
||||
// browser-only globals) - same reasoning as repos/** and examples/**
|
||||
// above. Pre-existing gap: this was blocking `npm run lint` (and thus
|
||||
// the pre-commit hook) repo-wide before this Electron work even
|
||||
// touched anything - see the electron-desktop branch's first commits.
|
||||
// Independent sub-packages, plus the generated staging dir
|
||||
// vnc/plugins/build/** that scripts/build-plugins.mjs writes the bundled
|
||||
// artifacts into (1.7 MB of vendored crypto - not our source to lint).
|
||||
"vnc/plugins/**",
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
@@ -20,8 +20,12 @@ interface ConfigData {
|
||||
faviconUrl: string;
|
||||
appLogoLightUrl: string;
|
||||
appLogoDarkUrl: string;
|
||||
appLogoLightUrlIsCustom: boolean;
|
||||
appLogoDarkUrlIsCustom: boolean;
|
||||
loginLogoLightUrl: string;
|
||||
loginLogoDarkUrl: string;
|
||||
loginLogoLightUrlIsCustom: boolean;
|
||||
loginLogoDarkUrlIsCustom: boolean;
|
||||
loginCompanyName: string;
|
||||
loginImprintUrl: string;
|
||||
loginPrivacyPolicyUrl: string;
|
||||
@@ -105,8 +109,12 @@ export function useConfig(): AppConfig {
|
||||
faviconUrl: configCache?.faviconUrl || '/branding/Bulwark_Favicon.svg',
|
||||
appLogoLightUrl: configCache?.appLogoLightUrl || '',
|
||||
appLogoDarkUrl: configCache?.appLogoDarkUrl || '',
|
||||
appLogoLightUrlIsCustom: configCache?.appLogoLightUrlIsCustom || false,
|
||||
appLogoDarkUrlIsCustom: configCache?.appLogoDarkUrlIsCustom || false,
|
||||
loginLogoLightUrl: configCache?.loginLogoLightUrl || '/branding/Bulwark_Logo_Color.svg',
|
||||
loginLogoDarkUrl: configCache?.loginLogoDarkUrl || '/branding/Bulwark_Logo_White.svg',
|
||||
loginLogoLightUrlIsCustom: configCache?.loginLogoLightUrlIsCustom || false,
|
||||
loginLogoDarkUrlIsCustom: configCache?.loginLogoDarkUrlIsCustom || false,
|
||||
loginCompanyName: configCache?.loginCompanyName || '',
|
||||
loginImprintUrl: configCache?.loginImprintUrl || '',
|
||||
loginPrivacyPolicyUrl: configCache?.loginPrivacyPolicyUrl || '',
|
||||
@@ -146,8 +154,12 @@ export function useConfig(): AppConfig {
|
||||
faviconUrl: configCache.faviconUrl,
|
||||
appLogoLightUrl: configCache.appLogoLightUrl,
|
||||
appLogoDarkUrl: configCache.appLogoDarkUrl,
|
||||
appLogoLightUrlIsCustom: configCache.appLogoLightUrlIsCustom,
|
||||
appLogoDarkUrlIsCustom: configCache.appLogoDarkUrlIsCustom,
|
||||
loginLogoLightUrl: configCache.loginLogoLightUrl,
|
||||
loginLogoDarkUrl: configCache.loginLogoDarkUrl,
|
||||
loginLogoLightUrlIsCustom: configCache.loginLogoLightUrlIsCustom,
|
||||
loginLogoDarkUrlIsCustom: configCache.loginLogoDarkUrlIsCustom,
|
||||
loginCompanyName: configCache.loginCompanyName,
|
||||
loginImprintUrl: configCache.loginImprintUrl,
|
||||
loginPrivacyPolicyUrl: configCache.loginPrivacyPolicyUrl,
|
||||
@@ -188,8 +200,12 @@ export function useConfig(): AppConfig {
|
||||
faviconUrl: data.faviconUrl,
|
||||
appLogoLightUrl: data.appLogoLightUrl,
|
||||
appLogoDarkUrl: data.appLogoDarkUrl,
|
||||
appLogoLightUrlIsCustom: data.appLogoLightUrlIsCustom,
|
||||
appLogoDarkUrlIsCustom: data.appLogoDarkUrlIsCustom,
|
||||
loginLogoLightUrl: data.loginLogoLightUrl,
|
||||
loginLogoDarkUrl: data.loginLogoDarkUrl,
|
||||
loginLogoLightUrlIsCustom: data.loginLogoLightUrlIsCustom,
|
||||
loginLogoDarkUrlIsCustom: data.loginLogoDarkUrlIsCustom,
|
||||
loginCompanyName: data.loginCompanyName,
|
||||
loginImprintUrl: data.loginImprintUrl,
|
||||
loginPrivacyPolicyUrl: data.loginPrivacyPolicyUrl,
|
||||
|
||||
@@ -43,6 +43,16 @@ migrateLegacyAdminLayout()
|
||||
}
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
// Install the first-party plugins this fork ships with (currently the
|
||||
// audited S/MIME plugin) into the server plugin registry - the same admin
|
||||
// channel an operator-uploaded ZIP lands in, so bundles still get
|
||||
// Ed25519-signed on serve and the privileged-tier gates still apply.
|
||||
// Staged by scripts/build-plugins.mjs; gated by the matching policy
|
||||
// feature toggle. Never throws.
|
||||
const { seedBundledPlugins } = await import("./lib/admin/bundled-plugins");
|
||||
await seedBundledPlugins();
|
||||
})
|
||||
.then(async () => {
|
||||
// Anonymous telemetry - on by default. Admins can disable via the
|
||||
// admin UI, the BULWARK_TELEMETRY env var, or by clearing the endpoint.
|
||||
|
||||
@@ -7,4 +7,5 @@ stalwart/stalwart-cli
|
||||
# Playwright/test artifacts
|
||||
node_modules/
|
||||
test-results/
|
||||
test-results-electron/
|
||||
playwright-report/
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { test, expect, _electron as electron } from '@playwright/test';
|
||||
import type { ElectronApplication, Page } from '@playwright/test';
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import { createServer } from 'node:net';
|
||||
import { get as httpGet } from 'node:http';
|
||||
import path from 'node:path';
|
||||
import { ACCOUNTS, JMAP_URL } from './helpers/config';
|
||||
import { sendMail } from './helpers/smtp';
|
||||
import { JmapClient } from './helpers/jmap';
|
||||
import { expectFolderUnread } from './helpers/app';
|
||||
|
||||
/**
|
||||
* Electron desktop shell against the real Stalwart fixture, end to end.
|
||||
*
|
||||
* Unlike e2e/electron-smoke.spec.ts (which calls window.vnc.showNotification
|
||||
* directly to prove the IPC bridge itself is wired), this launches the real
|
||||
* Electron shell, logs in as a real account against this same integration
|
||||
* stack's Stalwart, injects a message over SMTP exactly like
|
||||
* 02-mail-sync.spec.ts does for the browser-based suite, and asserts a
|
||||
* native notification fires as a side effect of the REAL push pipeline:
|
||||
*
|
||||
* SMTP delivery -> Stalwart -> JMAP StateChange push (lib/jmap/client.ts)
|
||||
* -> stores/email-store.ts's handleStateChange -> handleNewEmailNotification
|
||||
* -> app/(main)/[locale]/page.tsx's effect -> lib/electron-bridge.ts's
|
||||
* showElectronNotification() -> the contextBridge/IPC bridge
|
||||
* (electron/preload.ts) -> electron/main.ts's ipcMain.handle, which is
|
||||
* what actually shows the OS notification (and increments the
|
||||
* __notificationCallCount test hook this test polls).
|
||||
*
|
||||
* Nothing here is mocked - real SMTP socket, real Stalwart, real Electron
|
||||
* process, real IPC.
|
||||
*
|
||||
* WHY A DEV SERVER, NOT THE STANDALONE BUILD: electron/main.ts normally boots
|
||||
* the production "standalone" artifact (Phase 1 step 1), whose CSP
|
||||
* (proxy.ts) only allows TLS connections in production (`https:`/`wss:`).
|
||||
* This fixture's Stalwart is deliberately plain HTTP - the same reason
|
||||
* integration/webmail.Dockerfile runs the browser-suite's webmail in dev
|
||||
* mode instead of building it. This test makes the identical trade-off:
|
||||
* electron/main.ts's ELECTRON_LOAD_URL escape hatch (test-only, never used
|
||||
* by real users or any packaging/CI path) points the shell at a `next dev`
|
||||
* server this test spawns itself, instead of the standalone build. That
|
||||
* still exercises the real preload/IPC bridge, the real JMAP client
|
||||
* (identical source either way), and the real notification handler - the
|
||||
* only thing NOT covered here is the standalone-server-boot mechanism
|
||||
* itself, which e2e/electron-smoke.spec.ts already covers separately.
|
||||
*
|
||||
* NOTE on "the real WebSocket path": confirmed against the actual
|
||||
* `stalwartlabs/stalwart:v0.16` image this fixture runs (same as the
|
||||
* sandbox server this feature was built against) that its /jmap/ws endpoint
|
||||
* requires the same HTTP Authorization header as every other JMAP endpoint
|
||||
* on the WebSocket UPGRADE request itself - and confirmed separately that
|
||||
* the browser WebSocket API has no way to attach a custom header to that
|
||||
* handshake (a WHATWG spec restriction, not a CSP or Electron quirk - CSP
|
||||
* was a real, now-fixed blocker for reaching the network at all, see the
|
||||
* commit that added `wss:` to proxy.ts's production connect-src, but is not
|
||||
* why THIS specific handshake fails). So the WS attempt below will reach
|
||||
* the network correctly but still fail authentication against Stalwart
|
||||
* every time, and the client's circuit breaker (wsPermanentlyDisabled,
|
||||
* after 5 quick attempts) falls back to SSE within a few seconds. That
|
||||
* fallback is what actually delivers the push exercised below - a real,
|
||||
* working push path, just not literally the WebSocket one. Asserting the WS
|
||||
* handshake itself succeeds would be asserting something that cannot be
|
||||
* true against this server from a browser context; the assertion here is
|
||||
* on the thing that IS true end to end: a real delivery reaches the native
|
||||
* notification bridge no matter which transport carried the StateChange.
|
||||
*/
|
||||
const alice = ACCOUNTS.alice;
|
||||
const projectRoot = path.resolve(__dirname, '../..');
|
||||
|
||||
function getFreePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.unref();
|
||||
server.on('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (address && typeof address === 'object') {
|
||||
const { port } = address;
|
||||
server.close(() => resolve(port));
|
||||
} else {
|
||||
server.close(() => reject(new Error('Could not allocate a free localhost port')));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function waitForServerReady(url: string, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
return new Promise((resolve, reject) => {
|
||||
const attempt = () => {
|
||||
const req = httpGet(url, (res) => {
|
||||
res.resume();
|
||||
resolve();
|
||||
});
|
||||
req.on('error', () => {
|
||||
if (Date.now() > deadline) {
|
||||
reject(new Error(`Dev server never became reachable at ${url}`));
|
||||
return;
|
||||
}
|
||||
setTimeout(attempt, 300);
|
||||
});
|
||||
};
|
||||
attempt();
|
||||
});
|
||||
}
|
||||
|
||||
async function getNotificationCallCount(app: ElectronApplication): Promise<number> {
|
||||
return app.evaluate(({ app: electronApp }) => {
|
||||
const counters = electronApp as unknown as { __notificationCallCount?: number };
|
||||
return counters.__notificationCallCount ?? 0;
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('Electron desktop shell - real push notification', () => {
|
||||
test('a real SMTP delivery triggers the native notification bridge', async () => {
|
||||
const jmap = await JmapClient.connect(alice.email, alice.password);
|
||||
await jmap.reset();
|
||||
|
||||
const devPort = await getFreePort();
|
||||
const devUrl = `http://127.0.0.1:${devPort}`;
|
||||
|
||||
// `next dev` (not the standalone build - see the header comment above
|
||||
// for why) with JMAP_SERVER_URL pointed at this fixture's real Stalwart.
|
||||
const devServer: ChildProcess = spawn('npx', ['next', 'dev', '--turbopack', '-p', String(devPort)], {
|
||||
cwd: projectRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
JMAP_SERVER_URL: JMAP_URL,
|
||||
// Must be >= 32 chars (lib/impersonation/master-config.ts) - anything
|
||||
// shorter logs a "Failed to store Stalwart auth context" error on
|
||||
// every request. Not a real secret either way.
|
||||
SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
|
||||
NODE_ENV: 'development',
|
||||
},
|
||||
stdio: 'pipe',
|
||||
});
|
||||
devServer.stderr?.on('data', (chunk) => process.stderr.write(`[next dev] ${chunk}`));
|
||||
|
||||
let electronApp: ElectronApplication | undefined;
|
||||
try {
|
||||
// next dev's cold compile of the login route can take a while the
|
||||
// first time - generous timeout, matches this suite's overall 90s
|
||||
// test timeout with headroom for what comes after.
|
||||
await waitForServerReady(devUrl, 60000);
|
||||
|
||||
electronApp = await electron.launch({
|
||||
args: [projectRoot],
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_LOAD_URL: devUrl,
|
||||
},
|
||||
});
|
||||
|
||||
const appWindow: Page = await electronApp.firstWindow();
|
||||
await appWindow.waitForLoadState('domcontentloaded');
|
||||
// Diagnosing a failure locally: temporarily add
|
||||
// appWindow.on('console', (msg) => console.log(msg.type(), msg.text()));
|
||||
// appWindow.on('request', (req) => { if (/jmap/i.test(req.url())) console.log(req.method(), req.url()); });
|
||||
// right here - that's what surfaced the WS-then-SSE-fallback sequence
|
||||
// this test now relies on, and would surface the same for whatever
|
||||
// trips the retry below.
|
||||
|
||||
// Real login through the actual form - same selectors
|
||||
// integration/tests/helpers/app.ts's submitCredentials() uses. Not
|
||||
// reusing that helper directly because it also calls page.goto('/'),
|
||||
// which would navigate this window away from the dev server
|
||||
// electron/main.ts already loaded it against.
|
||||
await appWindow.locator('#username').waitFor({ state: 'visible', timeout: 30000 });
|
||||
await appWindow.fill('#username', alice.email);
|
||||
await appWindow.fill('#password', alice.password);
|
||||
await appWindow.click('button[type="submit"]');
|
||||
await appWindow.locator('[data-testid="account-switcher"]').first().waitFor({ state: 'visible', timeout: 30000 });
|
||||
|
||||
// The account switcher rendering only means the sidebar chrome is up,
|
||||
// not that the Inbox has actually loaded/been auto-selected yet - the
|
||||
// "new mail" notification only fires when handleStateChange's refresh
|
||||
// finds an actively-SELECTED inbox (stores/email-store.ts's
|
||||
// refreshCurrentMailbox() early-returns with no selectedMailbox).
|
||||
// Same wait 02-mail-sync.spec.ts's very first test uses right after
|
||||
// login, before its own first delivery, for exactly this reason.
|
||||
await expectFolderUnread(appWindow, { role: 'inbox' }, 0);
|
||||
|
||||
// Baseline before triggering delivery, so this assertion is robust
|
||||
// even if a stray notification fired during login/setup.
|
||||
const before = await getNotificationCallCount(electronApp);
|
||||
|
||||
const subject = `IT electron-push ${Date.now()}`;
|
||||
await sendMail({
|
||||
from: alice.email,
|
||||
authPass: alice.password,
|
||||
to: alice.email,
|
||||
subject,
|
||||
body: 'hi from the electron integration test',
|
||||
});
|
||||
|
||||
await expect
|
||||
.poll(() => getNotificationCallCount(electronApp!), {
|
||||
timeout: 60000,
|
||||
message: 'native notification bridge never fired after a real SMTP delivery',
|
||||
})
|
||||
.toBeGreaterThan(before);
|
||||
} finally {
|
||||
await electronApp?.close();
|
||||
devServer.kill();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,508 @@
|
||||
import { test, expect, _electron as electron } from '@playwright/test';
|
||||
import type { ElectronApplication, Page } from '@playwright/test';
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import { createServer } from 'node:net';
|
||||
import { get as httpGet } from 'node:http';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { ACCOUNTS, JMAP_URL } from './helpers/config';
|
||||
import { sendMail } from './helpers/smtp';
|
||||
import { JmapClient } from './helpers/jmap';
|
||||
import { expectFolderUnread } from './helpers/app';
|
||||
|
||||
/**
|
||||
* The encrypted local search index (lib/mail-index/**) against the real
|
||||
* Stalwart fixture. THREE tests, because no single configuration can cover the
|
||||
* whole feature - the reasons are specific and worth reading before changing
|
||||
* any of them.
|
||||
*
|
||||
* Constraint 1 - the renderer cannot reach this fixture from a production
|
||||
* build. The renderer talks JMAP DIRECTLY to Stalwart, and this fixture's
|
||||
* Stalwart is deliberately plain HTTP (integration/webmail.Dockerfile explains
|
||||
* why). The production CSP pins `connect-src` to `'self' https: wss:`. Setting
|
||||
* NODE_ENV=development at RUNTIME does not help: `next build` INLINES
|
||||
* process.env.NODE_ENV into the compiled middleware, so proxy.ts's `isDev` is
|
||||
* frozen at build time. Verified by watching a standalone server started with
|
||||
* NODE_ENV=development still serve the production CSP, and the login fail with
|
||||
* "Refused to connect ... violates connect-src 'self' https: wss:".
|
||||
*
|
||||
* Constraint 2 - the fd-3 key channel cannot survive `next dev`. `next dev`
|
||||
* forks its server process with an IPC channel that claims fd 3, so adopting it
|
||||
* fails with EEXIST; fd 4 in that process is not a pipe either (ENOTTY). Both
|
||||
* were observed, not assumed. Extra file descriptors simply are not plumbed
|
||||
* through `npx -> next dev -> forked server`. The real standalone server is a
|
||||
* single process and has no such problem (test 3 proves it).
|
||||
*
|
||||
* So each test takes the configuration that lets it prove its own half:
|
||||
*
|
||||
* 1. PIPELINE - drives the REAL standalone server over HTTP from Node, with a
|
||||
* real fd-3 key channel. CSP is irrelevant here because there is no
|
||||
* browser: a Node client with a real session cookie exercises the real
|
||||
* routes. This is the test that proves a real delivery becomes searchable
|
||||
* by a word from its BODY, and that the file on disk is really encrypted.
|
||||
*
|
||||
* 2. TRIGGER - proves the EVENT-DRIVEN wiring: a real SMTP delivery makes the
|
||||
* renderer POST /api/offline/reindex off the back of its live JMAP push.
|
||||
* Runs against `next dev` (constraint 1), and asserts the request is made -
|
||||
* the indexing itself is test 1's job.
|
||||
*
|
||||
* 3. WIRING - launches the REAL shell with no ELECTRON_LOAD_URL, so
|
||||
* electron/main.ts boots the real standalone artifact and stands up the real
|
||||
* fd-3 key service on real safeStorage. Asserts the index routes are
|
||||
* REACHABLE in a real build (401 "sign in", not 404 "feature absent", not
|
||||
* 503 "no native binding / no key channel").
|
||||
*
|
||||
* Nothing is mocked anywhere: real SMTP, real Stalwart, real Electron, real
|
||||
* SQLCipher, real safeStorage.
|
||||
*/
|
||||
const alice = ACCOUNTS.alice;
|
||||
const projectRoot = path.resolve(__dirname, '../..');
|
||||
|
||||
/** Mirrors lib/mail-index/paths.ts's accountFileToken(). */
|
||||
function accountFileToken(accountId: string): string {
|
||||
return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
function getFreePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.unref();
|
||||
server.on('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (address && typeof address === 'object') {
|
||||
const { port } = address;
|
||||
server.close(() => resolve(port));
|
||||
} else {
|
||||
server.close(() => reject(new Error('Could not allocate a free localhost port')));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function waitForServerReady(url: string, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
return new Promise((resolve, reject) => {
|
||||
const attempt = () => {
|
||||
const req = httpGet(url, (res) => {
|
||||
res.resume();
|
||||
resolve();
|
||||
});
|
||||
req.on('error', () => {
|
||||
if (Date.now() > deadline) {
|
||||
reject(new Error(`Server never became reachable at ${url}`));
|
||||
return;
|
||||
}
|
||||
setTimeout(attempt, 300);
|
||||
});
|
||||
};
|
||||
attempt();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Serves the key protocol of electron/key-service.ts over the child's inherited
|
||||
* fd. The key and the encryption are real; only safeStorage's wrapping of it is
|
||||
* out of the picture here, which is what test 3 covers.
|
||||
*/
|
||||
function serveKeyChannel(child: ChildProcess, fdIndex: number, key: Buffer): void {
|
||||
const channel = child.stdio[fdIndex] as NodeJS.ReadWriteStream | null;
|
||||
if (!channel) throw new Error(`child has no fd ${fdIndex} - key channel missing`);
|
||||
let buffer = '';
|
||||
channel.on('data', (chunk: Buffer) => {
|
||||
buffer += chunk.toString('utf8');
|
||||
let newline: number;
|
||||
while ((newline = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, newline);
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (!line.trim()) continue;
|
||||
const req = JSON.parse(line) as { id?: number; op?: string };
|
||||
const reply =
|
||||
req.op === 'getIndexKey'
|
||||
? { id: req.id, ok: true, key: key.toString('hex') }
|
||||
: req.op === 'deleteIndexKey'
|
||||
? { id: req.id, ok: true }
|
||||
: { id: req.id, ok: false, code: 'bad-request', error: 'unknown op' };
|
||||
channel.write(`${JSON.stringify(reply)}\n`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Minimal cookie jar - the index routes are cookie-authenticated. */
|
||||
class Jar {
|
||||
private cookies = new Map<string, string>();
|
||||
|
||||
absorb(response: Response): void {
|
||||
for (const raw of response.headers.getSetCookie()) {
|
||||
const [pair] = raw.split(';');
|
||||
const eq = pair.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
this.cookies.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
|
||||
}
|
||||
}
|
||||
|
||||
header(): string {
|
||||
return [...this.cookies].map(([k, v]) => `${k}=${v}`).join('; ');
|
||||
}
|
||||
}
|
||||
|
||||
interface SearchHit {
|
||||
contentType: string;
|
||||
id: string;
|
||||
title: string;
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
interface SearchResponse {
|
||||
ok?: boolean;
|
||||
count?: number;
|
||||
hits?: SearchHit[];
|
||||
contextBlock?: string;
|
||||
stats?: Array<{ contentType: string; count: number }>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
test.describe('Electron desktop shell - encrypted local search index', () => {
|
||||
test('pipeline: a real delivery becomes searchable by a body word, and the file is encrypted', async () => {
|
||||
const jmap = await JmapClient.connect(alice.email, alice.password);
|
||||
await jmap.reset();
|
||||
|
||||
const stamp = Date.now();
|
||||
const subject = `IT index subject ${stamp}`;
|
||||
// Appears ONLY in the body, so a hit proves the body was actually fetched
|
||||
// and indexed - not merely the subject, which any list view already holds.
|
||||
const bodyPhrase = `zurichlease${stamp}`;
|
||||
|
||||
// Deliver BEFORE indexing, so the catch-up path has something real to find.
|
||||
await sendMail({
|
||||
from: alice.email,
|
||||
authPass: alice.password,
|
||||
to: alice.email,
|
||||
subject,
|
||||
body: `Please review the ${bodyPhrase} renewal before September.`,
|
||||
});
|
||||
|
||||
const storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-index-it-'));
|
||||
const key = randomBytes(32);
|
||||
const port = await getFreePort();
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const serverEntry = path.join(projectRoot, '.next', 'standalone', 'server.js');
|
||||
expect(
|
||||
fs.existsSync(serverEntry),
|
||||
`missing ${serverEntry} - run "npm run build:standalone" first`,
|
||||
).toBe(true);
|
||||
|
||||
// The REAL standalone artifact, spawned exactly as electron/main.ts spawns
|
||||
// it (including the fd-3 key channel), just with plain node rather than
|
||||
// ELECTRON_RUN_AS_NODE - the server code is identical either way.
|
||||
const server = spawn(process.execPath, [serverEntry], {
|
||||
cwd: path.dirname(serverEntry),
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: String(port),
|
||||
HOSTNAME: '127.0.0.1',
|
||||
NODE_ENV: 'production',
|
||||
JMAP_SERVER_URL: JMAP_URL,
|
||||
SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
|
||||
VNCMAIL_DESKTOP_STORE_DIR: storeDir,
|
||||
VNCMAIL_DESKTOP_KEY_FD: '3',
|
||||
},
|
||||
stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
server.stderr?.on('data', (chunk) => process.stderr.write(`[standalone] ${chunk}`));
|
||||
serveKeyChannel(server, 3, key);
|
||||
|
||||
const jar = new Jar();
|
||||
const call = async (url: string, init?: RequestInit): Promise<Response> => {
|
||||
const response = await fetch(`${baseUrl}${url}`, {
|
||||
...init,
|
||||
headers: { ...(init?.headers ?? {}), cookie: jar.header() },
|
||||
});
|
||||
jar.absorb(response);
|
||||
return response;
|
||||
};
|
||||
const search = async (query: string, types?: string): Promise<SearchResponse> => {
|
||||
const params = new URLSearchParams({ q: query, stats: 'true' });
|
||||
if (types) params.set('types', types);
|
||||
const response = await call(`/api/offline/search?${params.toString()}`);
|
||||
if (!response.ok) return { error: `HTTP ${response.status}: ${await response.text()}` };
|
||||
return (await response.json()) as SearchResponse;
|
||||
};
|
||||
|
||||
try {
|
||||
await waitForServerReady(baseUrl, 60000);
|
||||
|
||||
// Server-side login. This route verifies the credentials against Stalwart
|
||||
// from Node and writes BOTH the session cookie and the jmap_stalwart_ctx
|
||||
// auth context the index routes read (app/api/auth/session/route.ts:94).
|
||||
const login = await call('/api/auth/session?slot=0', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
serverUrl: JMAP_URL,
|
||||
username: alice.email,
|
||||
password: alice.password,
|
||||
slot: 0,
|
||||
}),
|
||||
});
|
||||
expect(login.status, `login failed: ${await login.text()}`).toBe(200);
|
||||
|
||||
// The gate must be open and the native binding loaded, or every assertion
|
||||
// below would fail for an unrelated reason.
|
||||
const reachable = await call('/api/offline/search?stats=true&q=');
|
||||
expect(
|
||||
reachable.status,
|
||||
`index routes unreachable: ${(await reachable.text()).slice(0, 300)}`,
|
||||
).toBe(200);
|
||||
|
||||
// Index it. This is the catch-up shape (no ids), which is what the app
|
||||
// runs at launch.
|
||||
const reindex = await call('/api/offline/reindex', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ catchUp: true }),
|
||||
});
|
||||
const reindexBody = await reindex.json();
|
||||
expect(reindex.status, JSON.stringify(reindexBody)).toBe(200);
|
||||
expect(
|
||||
reindexBody.written?.mail,
|
||||
`no mail indexed: ${JSON.stringify(reindexBody)}`,
|
||||
).toBeGreaterThan(0);
|
||||
|
||||
// THE assertion: found by a word that exists only in the message body.
|
||||
const hit = await search(bodyPhrase);
|
||||
expect(hit.error).toBeUndefined();
|
||||
expect(hit.count, `search for a body word found nothing: ${JSON.stringify(hit)}`)
|
||||
.toBeGreaterThan(0);
|
||||
expect(hit.hits?.[0].contentType).toBe('mail');
|
||||
expect(hit.hits?.[0].title).toBe(subject);
|
||||
expect(hit.hits?.[0].snippet).toContain(bodyPhrase);
|
||||
// The prompt-ready retrieval surface an AI feature would consume.
|
||||
expect(hit.contextBlock).toContain('[EMAIL]');
|
||||
expect(hit.contextBlock).toContain(subject);
|
||||
|
||||
// Also findable by sender address, which lives in the `people` column.
|
||||
expect((await search(alice.email)).count).toBeGreaterThan(0);
|
||||
|
||||
// Type filtering must filter, and a word in no message must not match -
|
||||
// otherwise the hit above proves nothing about relevance.
|
||||
expect((await search(bodyPhrase, 'calendar')).count).toBe(0);
|
||||
expect((await search(bodyPhrase, 'mail')).count).toBeGreaterThan(0);
|
||||
expect((await search(`absent${stamp}`)).count).toBe(0);
|
||||
|
||||
// Catch-up must be idempotent: a second pass must not duplicate rows.
|
||||
const before = ((await search(bodyPhrase)).stats ?? [])
|
||||
.find((s) => s.contentType === 'mail')?.count ?? 0;
|
||||
const second = await call('/api/offline/reindex', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ catchUp: true }),
|
||||
});
|
||||
expect(second.status).toBe(200);
|
||||
const after = ((await search(bodyPhrase)).stats ?? [])
|
||||
.find((s) => s.contentType === 'mail')?.count ?? 0;
|
||||
expect(after).toBe(before);
|
||||
expect((await search(bodyPhrase)).count).toBe(1);
|
||||
|
||||
// Calendar/contacts/files: assert they were ATTEMPTED and did not error,
|
||||
// rather than asserting counts - this fixture provisions mailboxes only,
|
||||
// so an empty calendar is the correct result and a count assertion would
|
||||
// be testing the fixture rather than the code.
|
||||
const errors = (reindexBody.errors ?? []) as Array<{ contentType: string; message: string }>;
|
||||
expect(errors, `per-type failures during reindex: ${JSON.stringify(errors)}`).toEqual([]);
|
||||
const attempted = Object.keys(reindexBody.written ?? {});
|
||||
const skipped = (reindexBody.skipped ?? []) as string[];
|
||||
expect(
|
||||
[...attempted, ...skipped].sort(),
|
||||
'every content type must be either attempted or explicitly skipped',
|
||||
).toEqual(['calendar', 'contact', 'file', 'mail']);
|
||||
} finally {
|
||||
server.kill();
|
||||
// Let the process release its WAL files before reading them.
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
|
||||
// ── the file on disk is genuinely encrypted ──────────────────────────────
|
||||
const accountId = `${alice.email}@${new URL(JMAP_URL).hostname}`;
|
||||
const dbPath = path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`);
|
||||
expect(fs.existsSync(dbPath), `no index database at ${dbPath}`).toBe(true);
|
||||
|
||||
// Read every file the store wrote, WAL included: the newest rows can still
|
||||
// be sitting in the -wal, so checking only the main database could miss
|
||||
// plaintext that is genuinely on disk.
|
||||
const onDisk = Buffer.concat(
|
||||
['', '-wal', '-shm']
|
||||
.map((suffix) => `${dbPath}${suffix}`)
|
||||
.filter((f) => fs.existsSync(f))
|
||||
.map((f) => fs.readFileSync(f)),
|
||||
);
|
||||
expect(onDisk.length).toBeGreaterThan(0);
|
||||
// The assertions that catch a silently-UNENCRYPTED store. `PRAGMA key` is a
|
||||
// no-op on a non-SQLCipher binding - no error, working database, mailbox in
|
||||
// cleartext - so every functional assertion above would pass either way.
|
||||
expect(
|
||||
fs.readFileSync(dbPath).subarray(0, 15).toString('latin1'),
|
||||
'the index file has a plain SQLite header - it is NOT encrypted',
|
||||
).not.toBe('SQLite format 3');
|
||||
expect(
|
||||
onDisk.includes(bodyPhrase),
|
||||
'the message body is recoverable from the raw database bytes - not encrypted',
|
||||
).toBe(false);
|
||||
expect(
|
||||
onDisk.includes(subject),
|
||||
'the subject is recoverable from the raw database bytes - not encrypted',
|
||||
).toBe(false);
|
||||
|
||||
fs.rmSync(storeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('trigger: a real delivery makes the renderer ask the index to update', async () => {
|
||||
const jmap = await JmapClient.connect(alice.email, alice.password);
|
||||
await jmap.reset();
|
||||
|
||||
const devPort = await getFreePort();
|
||||
const devUrl = `http://127.0.0.1:${devPort}`;
|
||||
const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-trigger-profile-'));
|
||||
|
||||
// `next dev` for the CSP reason in the header comment. No key channel here:
|
||||
// this test asserts the REQUEST is made, which is the wiring it owns; the
|
||||
// indexing itself is test 1's job. (Extra fds don't survive next dev
|
||||
// anyway - constraint 2 above.)
|
||||
const devServer = spawn('npx', ['next', 'dev', '--turbopack', '-p', String(devPort)], {
|
||||
cwd: projectRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
JMAP_SERVER_URL: JMAP_URL,
|
||||
SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
|
||||
NODE_ENV: 'development',
|
||||
// Enough for the route to exist and pass its gate; it fails later on the
|
||||
// absent key channel, which this test deliberately does not assert on.
|
||||
VNCMAIL_DESKTOP_STORE_DIR: path.join(userDataDir, 'offline'),
|
||||
VNCMAIL_DESKTOP_KEY_FD: '3',
|
||||
},
|
||||
stdio: 'pipe',
|
||||
});
|
||||
devServer.stderr?.on('data', (chunk) => process.stderr.write(`[next dev] ${chunk}`));
|
||||
|
||||
let electronApp: ElectronApplication | undefined;
|
||||
try {
|
||||
await waitForServerReady(devUrl, 90000);
|
||||
|
||||
electronApp = await electron.launch({
|
||||
args: [projectRoot, `--user-data-dir=${userDataDir}`],
|
||||
env: { ...process.env, ELECTRON_LOAD_URL: devUrl },
|
||||
});
|
||||
|
||||
const appWindow: Page = await electronApp.firstWindow();
|
||||
await appWindow.waitForLoadState('domcontentloaded');
|
||||
|
||||
await appWindow.locator('#username').waitFor({ state: 'visible', timeout: 60000 });
|
||||
await appWindow.fill('#username', alice.email);
|
||||
await appWindow.fill('#password', alice.password);
|
||||
await appWindow.click('button[type="submit"]');
|
||||
await appWindow
|
||||
.locator('[data-testid="account-switcher"]')
|
||||
.first()
|
||||
.waitFor({ state: 'visible', timeout: 60000 });
|
||||
|
||||
// An actively-selected inbox is a precondition for the push handler's
|
||||
// refresh, which is what schedules the index update - the same reason
|
||||
// 11-electron-notification.spec.ts waits here.
|
||||
await expectFolderUnread(appWindow, { role: 'inbox' }, 0);
|
||||
|
||||
const reindexCalls: string[] = [];
|
||||
appWindow.on('request', (request) => {
|
||||
if (request.method() === 'POST' && request.url().includes('/api/offline/reindex')) {
|
||||
reindexCalls.push(request.postData() ?? '');
|
||||
}
|
||||
});
|
||||
// Let the launch-time catch-up land first so it is not mistaken for the
|
||||
// delivery-driven call below.
|
||||
await appWindow.waitForTimeout(8000);
|
||||
const baseline = reindexCalls.length;
|
||||
|
||||
await sendMail({
|
||||
from: alice.email,
|
||||
authPass: alice.password,
|
||||
to: alice.email,
|
||||
subject: `IT index trigger ${Date.now()}`,
|
||||
body: 'a delivery should make the renderer ask the index to update',
|
||||
});
|
||||
|
||||
await expect
|
||||
.poll(() => reindexCalls.length, {
|
||||
timeout: 60000,
|
||||
message:
|
||||
'a real delivery did not make the renderer POST /api/offline/reindex - ' +
|
||||
'the push -> handleStateChange -> indexOnStateChange wiring is broken',
|
||||
})
|
||||
.toBeGreaterThan(baseline);
|
||||
|
||||
// The delivery-driven call must name the mail type, rather than being an
|
||||
// unconditional full catch-up.
|
||||
const triggered = reindexCalls.slice(baseline);
|
||||
expect(
|
||||
triggered.some((body) => body.includes('"mail"')),
|
||||
`no reindex call mentioned the mail type: ${JSON.stringify(triggered)}`,
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await electronApp?.close();
|
||||
devServer.kill();
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('wiring: the real standalone boot reaches the index with a real safeStorage key', async () => {
|
||||
// A FRESH profile is load-bearing, not hygiene: the 401 this test asserts is
|
||||
// "nobody is signed in", and a leftover jmap_stalwart_ctx cookie from any
|
||||
// previous run turns it into a 200. That actually happened while writing this.
|
||||
const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-wiring-profile-'));
|
||||
const electronApp = await electron.launch({
|
||||
args: [projectRoot, `--user-data-dir=${userDataDir}`],
|
||||
env: {
|
||||
...process.env,
|
||||
JMAP_SERVER_URL: JMAP_URL,
|
||||
SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const appWindow: Page = await electronApp.firstWindow();
|
||||
await appWindow.waitForLoadState('domcontentloaded');
|
||||
await appWindow.locator('#username').waitFor({ state: 'visible', timeout: 60000 });
|
||||
|
||||
// safeStorage must be usable, or main.ts deliberately refuses to enable
|
||||
// the feature at all (electron/key-service.ts's checkEncryptionAvailable).
|
||||
const encryptionAvailable = await electronApp.evaluate(({ safeStorage }) =>
|
||||
safeStorage.isEncryptionAvailable(),
|
||||
);
|
||||
expect(
|
||||
encryptionAvailable,
|
||||
'safeStorage reports no encryption available on this host, so main.ts ' +
|
||||
'correctly disabled the index - this assertion cannot pass here',
|
||||
).toBe(true);
|
||||
|
||||
const probe = await appWindow.evaluate(async () => {
|
||||
const response = await fetch('/api/offline/search?q=anything');
|
||||
return { status: response.status, body: (await response.text()).slice(0, 300) };
|
||||
});
|
||||
|
||||
// 401 = the gate opened, the native binding loaded and the fd-3 key
|
||||
// channel is present; it refuses only because nobody is signed in (this
|
||||
// build cannot log in against a plain-HTTP Stalwart - constraint 1).
|
||||
// 404 => VNCMAIL_DESKTOP_STORE_DIR was never set (gate closed, or
|
||||
// main.ts refused because no OS keyring is available)
|
||||
// 503 => the native binding or the key channel is missing from the real
|
||||
// artifact - the class of failure only a real build reveals
|
||||
expect(
|
||||
probe.status,
|
||||
`expected 401 (reachable, unauthenticated) but got ${probe.status}: ${probe.body}`,
|
||||
).toBe(401);
|
||||
} finally {
|
||||
await electronApp.close();
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,600 @@
|
||||
import { test, expect, _electron as electron } from '@playwright/test';
|
||||
import type { ElectronApplication, Page } from '@playwright/test';
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import { createServer } from 'node:net';
|
||||
import { get as httpGet } from 'node:http';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { ACCOUNTS, JMAP_URL } from './helpers/config';
|
||||
import { sendMail } from './helpers/smtp';
|
||||
import { JmapClient } from './helpers/jmap';
|
||||
|
||||
/**
|
||||
* The offline mail replica (lib/offline-replica/**) against the real Stalwart
|
||||
* fixture, with a REAL NETWORK CUT.
|
||||
*
|
||||
* THE POINT OF THIS FILE: a sync test that never tests the offline case has not
|
||||
* tested the feature. So test 1 syncs against a live server, then makes the
|
||||
* backend genuinely unreachable, and only then asserts that a previously-synced
|
||||
* message still returns its full HTML body - from the encrypted replica, with no
|
||||
* network available to fall back to.
|
||||
*
|
||||
* HOW THE CUT IS MADE. The standalone server is started with
|
||||
* `JMAP_SERVER_URL` pointing at a LOCAL PROXY that forwards to Stalwart. Killing
|
||||
* the proxy's listener makes every JMAP request fail with ECONNREFUSED - a real
|
||||
* transport failure at the socket level, not a mock, not a stubbed fetch, and not
|
||||
* a flag the code under test can see. Preferred over stopping the Stalwart
|
||||
* container because it cuts only THIS test's path and leaves the shared fixture
|
||||
* (and any concurrently-running suite) untouched.
|
||||
*
|
||||
* THE SAME TWO CONSTRAINTS as 12-electron-mail-index.spec.ts apply and are why
|
||||
* this is split into two tests rather than one:
|
||||
*
|
||||
* 1. The RENDERER cannot reach this fixture from a production build. It talks
|
||||
* JMAP directly to Stalwart, which here is deliberately plain HTTP, and the
|
||||
* production CSP pins `connect-src` to `'self' https: wss:`. NODE_ENV at
|
||||
* runtime does not help - `next build` inlines it into the middleware.
|
||||
* 2. The fd-3 key channel cannot survive `next dev`, which claims fd 3 for its
|
||||
* own IPC. So the two configurations are mutually exclusive: a real key
|
||||
* channel means no browser, a browser means no key channel.
|
||||
*
|
||||
* Test 1 therefore drives the REAL standalone server over HTTP from Node with a
|
||||
* real fd-3 key channel - no browser needed, because the routes are the thing
|
||||
* being proven. Test 2 launches the REAL Electron shell to prove the routes exist
|
||||
* and are reachable in a genuine build, which is the class of failure only a real
|
||||
* build reveals (the standalone output silently dropping a native prebuild, say).
|
||||
*
|
||||
* WHAT THIS FILE DOES NOT PROVE: that `components/email/email-viewer.tsx` paints
|
||||
* the replica-served body in a browser while offline. That needs a renderer, a
|
||||
* key channel and a reachable-then-unreachable JMAP server simultaneously, which
|
||||
* constraints 1 and 2 make impossible against this fixture. The read path returns
|
||||
* a field-for-field `Email` (asserted below, including `bodyValues` keyed by the
|
||||
* same partIds as `htmlBody`), and the renderer-side gate is covered by
|
||||
* `lib/__tests__/offline-fallback-client.test.ts` - but the final paint is NOT
|
||||
* covered by a real offline browser run. Stated rather than implied.
|
||||
*/
|
||||
const alice = ACCOUNTS.alice;
|
||||
const projectRoot = path.resolve(__dirname, '../..');
|
||||
|
||||
/** Mirrors lib/mail-index/paths.ts's accountFileToken(). */
|
||||
function accountFileToken(accountId: string): string {
|
||||
return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
function getFreePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.unref();
|
||||
server.on('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (address && typeof address === 'object') {
|
||||
const { port } = address;
|
||||
server.close(() => resolve(port));
|
||||
} else {
|
||||
server.close(() => reject(new Error('Could not allocate a free localhost port')));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function waitForServerReady(url: string, timeoutMs: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
return new Promise((resolve, reject) => {
|
||||
const attempt = () => {
|
||||
const req = httpGet(url, (res) => {
|
||||
res.resume();
|
||||
resolve();
|
||||
});
|
||||
req.on('error', () => {
|
||||
if (Date.now() > deadline) {
|
||||
reject(new Error(`Server never became reachable at ${url}`));
|
||||
return;
|
||||
}
|
||||
setTimeout(attempt, 300);
|
||||
});
|
||||
};
|
||||
attempt();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A raw TCP forwarder in front of Stalwart, so the test can sever the backend at
|
||||
* the socket level. `cut()` closes the listener AND destroys every live socket, so
|
||||
* a pooled keep-alive connection cannot keep working after the cut.
|
||||
*/
|
||||
async function startCuttableProxy(target: { host: string; port: number }): Promise<{
|
||||
port: number;
|
||||
cut: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
}> {
|
||||
const { connect } = await import('node:net');
|
||||
const sockets = new Set<import('node:net').Socket>();
|
||||
const server = createServer((incoming) => {
|
||||
sockets.add(incoming);
|
||||
incoming.on('close', () => sockets.delete(incoming));
|
||||
incoming.on('error', () => incoming.destroy());
|
||||
const upstream = connect(target.port, target.host, () => {
|
||||
incoming.pipe(upstream);
|
||||
upstream.pipe(incoming);
|
||||
});
|
||||
sockets.add(upstream);
|
||||
upstream.on('close', () => sockets.delete(upstream));
|
||||
upstream.on('error', () => { incoming.destroy(); upstream.destroy(); });
|
||||
});
|
||||
const port = await getFreePort();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(port, '127.0.0.1', () => resolve());
|
||||
});
|
||||
|
||||
const closeAll = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
for (const s of sockets) s.destroy();
|
||||
sockets.clear();
|
||||
server.close(() => resolve());
|
||||
// `close()` only stops new connections; the destroys above handle the rest.
|
||||
setTimeout(resolve, 500);
|
||||
});
|
||||
|
||||
return { port, cut: closeAll, stop: closeAll };
|
||||
}
|
||||
|
||||
/**
|
||||
* Serves the key protocol of electron/key-service.ts over the child's inherited
|
||||
* fd. The key and the encryption are real; only safeStorage's wrapping of it is
|
||||
* out of the picture here, which is what test 2 covers.
|
||||
*/
|
||||
function serveKeyChannel(child: ChildProcess, fdIndex: number, key: Buffer): void {
|
||||
const channel = child.stdio[fdIndex] as NodeJS.ReadWriteStream | null;
|
||||
if (!channel) throw new Error(`child has no fd ${fdIndex} - key channel missing`);
|
||||
let buffer = '';
|
||||
channel.on('data', (chunk: Buffer) => {
|
||||
buffer += chunk.toString('utf8');
|
||||
let newline: number;
|
||||
while ((newline = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, newline);
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (!line.trim()) continue;
|
||||
const req = JSON.parse(line) as { id?: number; op?: string };
|
||||
const reply =
|
||||
req.op === 'getIndexKey'
|
||||
? { id: req.id, ok: true, key: key.toString('hex') }
|
||||
: req.op === 'deleteIndexKey'
|
||||
? { id: req.id, ok: true }
|
||||
: { id: req.id, ok: false, code: 'bad-request', error: 'unknown op' };
|
||||
channel.write(`${JSON.stringify(reply)}\n`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
class Jar {
|
||||
private cookies = new Map<string, string>();
|
||||
absorb(response: Response): void {
|
||||
for (const raw of response.headers.getSetCookie()) {
|
||||
const [pair] = raw.split(';');
|
||||
const eq = pair.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
this.cookies.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
|
||||
}
|
||||
}
|
||||
header(): string {
|
||||
return [...this.cookies].map(([k, v]) => `${k}=${v}`).join('; ');
|
||||
}
|
||||
}
|
||||
|
||||
interface CycleReport {
|
||||
ok: boolean;
|
||||
unfinishedWork: boolean;
|
||||
bootstrapped: boolean;
|
||||
envelopesWritten: number;
|
||||
bodiesWritten: number;
|
||||
envelopesDeleted: number;
|
||||
coveragePhase: string;
|
||||
resyncRequired: boolean;
|
||||
warnings: string[];
|
||||
error?: string;
|
||||
errorClass?: string;
|
||||
}
|
||||
|
||||
test.describe('Electron desktop shell - offline mail replica', () => {
|
||||
test('syncs full bodies, then serves a synced message with the backend UNREACHABLE', async () => {
|
||||
test.setTimeout(240_000);
|
||||
|
||||
const jmap = await JmapClient.connect(alice.email, alice.password);
|
||||
await jmap.reset();
|
||||
|
||||
const stamp = Date.now();
|
||||
const subject = `IT replica subject ${stamp}`;
|
||||
// Appears ONLY in the HTML body, so a hit proves the full body was stored -
|
||||
// not the preview or the subject, which any envelope already carries.
|
||||
const bodyPhrase = `luzernrenewal${stamp}`;
|
||||
const htmlMarker = `<strong>${bodyPhrase}</strong>`;
|
||||
|
||||
await sendMail({
|
||||
from: alice.email,
|
||||
authPass: alice.password,
|
||||
to: alice.email,
|
||||
subject,
|
||||
body: `plain text ${bodyPhrase}`,
|
||||
html: `<html><body><p>Please review the ${htmlMarker} before September.</p></body></html>`,
|
||||
});
|
||||
// A second message, so "the list came from the replica" is not a one-row
|
||||
// coincidence.
|
||||
await sendMail({
|
||||
from: alice.email,
|
||||
authPass: alice.password,
|
||||
to: alice.email,
|
||||
subject: `IT replica second ${stamp}`,
|
||||
body: 'the second message',
|
||||
});
|
||||
|
||||
const storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-it-'));
|
||||
const key = randomBytes(32);
|
||||
const stalwart = new URL(JMAP_URL);
|
||||
const proxy = await startCuttableProxy({
|
||||
host: stalwart.hostname,
|
||||
port: Number(stalwart.port || 80),
|
||||
});
|
||||
const proxiedJmapUrl = `http://127.0.0.1:${proxy.port}`;
|
||||
|
||||
const port = await getFreePort();
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const serverEntry = path.join(projectRoot, '.next', 'standalone', 'server.js');
|
||||
expect(
|
||||
fs.existsSync(serverEntry),
|
||||
`missing ${serverEntry} - run "npm run build:standalone" first`,
|
||||
).toBe(true);
|
||||
|
||||
const server = spawn(process.execPath, [serverEntry], {
|
||||
cwd: path.dirname(serverEntry),
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: String(port),
|
||||
HOSTNAME: '127.0.0.1',
|
||||
NODE_ENV: 'production',
|
||||
// Through the cuttable proxy, so the backend can be severed later.
|
||||
JMAP_SERVER_URL: proxiedJmapUrl,
|
||||
SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
|
||||
VNCMAIL_DESKTOP_STORE_DIR: storeDir,
|
||||
VNCMAIL_DESKTOP_KEY_FD: '3',
|
||||
},
|
||||
stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
server.stderr?.on('data', (chunk) => process.stderr.write(`[standalone] ${chunk}`));
|
||||
serveKeyChannel(server, 3, key);
|
||||
|
||||
const jar = new Jar();
|
||||
const call = async (url: string, init?: RequestInit): Promise<Response> => {
|
||||
const response = await fetch(`${baseUrl}${url}`, {
|
||||
...init,
|
||||
headers: { ...(init?.headers ?? {}), cookie: jar.header() },
|
||||
});
|
||||
jar.absorb(response);
|
||||
return response;
|
||||
};
|
||||
const sync = async (body: Record<string, unknown> = {}): Promise<CycleReport> => {
|
||||
const response = await call('/api/offline/sync', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const parsed = await response.json();
|
||||
expect(response.status, JSON.stringify(parsed)).toBe(200);
|
||||
return parsed.report as CycleReport;
|
||||
};
|
||||
|
||||
try {
|
||||
await waitForServerReady(baseUrl, 90_000);
|
||||
|
||||
const login = await call('/api/auth/session?slot=0', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
serverUrl: proxiedJmapUrl,
|
||||
username: alice.email,
|
||||
password: alice.password,
|
||||
slot: 0,
|
||||
}),
|
||||
});
|
||||
const loginText = await login.text();
|
||||
expect(login.status, `login failed: ${loginText}`).toBe(200);
|
||||
|
||||
// The gate must be open and the native binding loaded, or every assertion
|
||||
// below would fail for an unrelated reason.
|
||||
const reachable = await call('/api/offline/status');
|
||||
const reachableText = await reachable.text();
|
||||
expect(
|
||||
reachable.status,
|
||||
`replica routes unreachable: ${reachableText.slice(0, 400)}`,
|
||||
).toBe(200);
|
||||
|
||||
// ── ONLINE: bootstrap, then chain until the cycle reports itself done ──
|
||||
const first = await sync();
|
||||
expect(first.error, `first cycle failed: ${first.error}`).toBeUndefined();
|
||||
expect(first.bootstrapped, 'the first cycle must bootstrap').toBe(true);
|
||||
|
||||
let report = first;
|
||||
for (let i = 0; i < 12 && report.unfinishedWork; i++) report = await sync();
|
||||
expect(
|
||||
report.unfinishedWork,
|
||||
`sync never settled: ${JSON.stringify(report)}`,
|
||||
).toBe(false);
|
||||
// Termination is a real property here: the body-queue give-up marks and the
|
||||
// inserted-not-attempted count are what stop this looping forever.
|
||||
expect(report.coveragePhase).toBe('complete');
|
||||
expect(report.resyncRequired).toBe(false);
|
||||
|
||||
const status = await (await call('/api/offline/status')).json();
|
||||
expect(status.synced).toBe(true);
|
||||
expect(
|
||||
status.stats.envelopes,
|
||||
`no envelopes stored: ${JSON.stringify(status.stats)}`,
|
||||
).toBeGreaterThanOrEqual(2);
|
||||
expect(
|
||||
status.stats.bodies,
|
||||
`no BODIES stored - the replica would be no better than the search index`,
|
||||
).toBeGreaterThanOrEqual(2);
|
||||
expect(status.stats.mailboxes).toBeGreaterThan(0);
|
||||
|
||||
// ── THE DELTA PATH: a message that arrives AFTER the cursor was captured ──
|
||||
// Bootstrap alone would satisfy every assertion below, so this is what actually
|
||||
// exercises `Email/changes` and proves the stored cursor is USABLE rather than
|
||||
// merely present. It is also the assertion that fails if an `Email/get` state
|
||||
// token is ever adopted as a `/changes` cursor: the fast-forwarded cursor
|
||||
// reports no changes, and this message never arrives.
|
||||
const deltaSubject = `IT replica delta ${stamp}`;
|
||||
const deltaPhrase = `bernrenewal${stamp}`;
|
||||
await sendMail({
|
||||
from: alice.email,
|
||||
authPass: alice.password,
|
||||
to: alice.email,
|
||||
subject: deltaSubject,
|
||||
body: `plain ${deltaPhrase}`,
|
||||
html: `<html><body><p>delta ${deltaPhrase}</p></body></html>`,
|
||||
});
|
||||
|
||||
let delta = await sync();
|
||||
for (let i = 0; i < 10 && (delta.unfinishedWork || delta.envelopesWritten === 0); i++) {
|
||||
delta = await sync();
|
||||
}
|
||||
expect(delta.bootstrapped, 'the delta cycle must NOT re-bootstrap').toBe(false);
|
||||
const afterDelta = await (await call('/api/offline/status')).json();
|
||||
expect(
|
||||
afterDelta.stats.envelopes,
|
||||
`Email/changes did not deliver a message that arrived after the cursor was ` +
|
||||
`captured: ${JSON.stringify(afterDelta.stats)}`,
|
||||
).toBeGreaterThanOrEqual(3);
|
||||
expect(
|
||||
afterDelta.stats.bodies,
|
||||
'the delta path delivered the envelope but never queued its body',
|
||||
).toBeGreaterThanOrEqual(3);
|
||||
expect(afterDelta.resyncRequired, 'a healthy delta cycle must not invalidate a cursor').toBe(false);
|
||||
|
||||
// Find the message and its mailbox while still online, so the offline phase
|
||||
// asserts on known ids rather than discovering them from the thing under test.
|
||||
const mailboxesOnline = await (await call('/api/offline/mail?kind=mailboxes')).json();
|
||||
const inbox = (mailboxesOnline.mailboxes as Array<{ id: string; role?: string }>)
|
||||
.find((m) => m.role === 'inbox');
|
||||
expect(inbox, 'the replica holds no inbox').toBeTruthy();
|
||||
|
||||
const listOnline = await (
|
||||
await call(`/api/offline/mail?kind=list&mailboxId=${encodeURIComponent(inbox!.id)}&limit=50`)
|
||||
).json();
|
||||
const target = (listOnline.emails as Array<{ id: string; subject?: string }>)
|
||||
.find((e) => e.subject === subject);
|
||||
expect(target, `the synced message is not in the replica: ${JSON.stringify(listOnline.emails?.map((e: {subject?: string}) => e.subject))}`).toBeTruthy();
|
||||
|
||||
// ── THE CUT: sever the backend at the socket level ────────────────────
|
||||
await proxy.cut();
|
||||
|
||||
// Prove the cut is real, from inside the server process's own network
|
||||
// namespace: a live JMAP call must now fail. `/api/offline/sync` reaches
|
||||
// Stalwart first thing, so it is the honest probe.
|
||||
const afterCut = await call('/api/offline/sync', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
const afterCutBody = await afterCut.json();
|
||||
expect(
|
||||
afterCut.status,
|
||||
`the backend is still reachable, so the offline assertions below would prove nothing: ` +
|
||||
`${JSON.stringify(afterCutBody)}`,
|
||||
).not.toBe(200);
|
||||
|
||||
// ── OFFLINE: the actual feature ───────────────────────────────────────
|
||||
const messageResponse = await call(
|
||||
`/api/offline/mail?kind=message&id=${encodeURIComponent(target!.id)}`,
|
||||
);
|
||||
// Read the body ONCE: `expect`'s message argument is evaluated eagerly, so
|
||||
// putting `await response.text()` in it consumes the stream before .json().
|
||||
const messageText = await messageResponse.text();
|
||||
expect(
|
||||
messageResponse.status,
|
||||
`the offline read path failed with the backend down: ${messageText.slice(0, 400)}`,
|
||||
).toBe(200);
|
||||
const offline = JSON.parse(messageText);
|
||||
expect(offline.available).toBe(true);
|
||||
expect(offline.hasBody, 'the message has no stored body offline').toBe(true);
|
||||
|
||||
const email = offline.email as {
|
||||
id: string; subject?: string; receivedAt: string;
|
||||
htmlBody?: Array<{ partId: string; type: string }>;
|
||||
textBody?: Array<{ partId: string }>;
|
||||
bodyValues?: Record<string, { value: string }>;
|
||||
from?: Array<{ email: string }>;
|
||||
keywords?: Record<string, boolean>;
|
||||
mailboxIds?: Record<string, boolean>;
|
||||
headers?: Record<string, string | string[]>;
|
||||
};
|
||||
|
||||
expect(email.id).toBe(target!.id);
|
||||
expect(email.subject).toBe(subject);
|
||||
|
||||
// THE ASSERTION: the full HTML body, recovered with no network.
|
||||
const htmlPartId = email.htmlBody?.[0]?.partId;
|
||||
expect(htmlPartId, 'no htmlBody part offline').toBeTruthy();
|
||||
const html = email.bodyValues?.[htmlPartId as string]?.value ?? '';
|
||||
expect(
|
||||
html,
|
||||
'the HTML body is not in the replica - this is the whole feature',
|
||||
).toContain(htmlMarker);
|
||||
expect(html).toContain(bodyPhrase);
|
||||
|
||||
// `bodyValues` MUST be keyed by the same partIds as htmlBody/textBody, or
|
||||
// email-viewer.tsx's isBodyLoading gate sits on its skeleton forever
|
||||
// (hasBodyParts true, bodyValues unusable).
|
||||
for (const part of [...(email.htmlBody ?? []), ...(email.textBody ?? [])]) {
|
||||
expect(
|
||||
email.bodyValues?.[part.partId],
|
||||
`bodyValues is missing partId ${part.partId}, which the viewer requires`,
|
||||
).toBeTruthy();
|
||||
}
|
||||
|
||||
// The rest of the shape the renderer reads.
|
||||
expect(email.from?.[0]?.email).toBe(alice.email);
|
||||
expect(email.receivedAt).toBeTruthy();
|
||||
expect(Object.keys(email.mailboxIds ?? {})).toContain(inbox!.id);
|
||||
// Header normalisation happened server-side (the array -> record flattening
|
||||
// the online path does in parseEmailHeaders).
|
||||
expect(email.headers && !Array.isArray(email.headers)).toBe(true);
|
||||
|
||||
// The list and the folder tree must also survive the cut.
|
||||
const listOffline = await (
|
||||
await call(`/api/offline/mail?kind=list&mailboxId=${encodeURIComponent(inbox!.id)}&limit=50`)
|
||||
).json();
|
||||
expect(listOffline.available).toBe(true);
|
||||
expect(listOffline.emails.length).toBeGreaterThanOrEqual(2);
|
||||
expect(
|
||||
(listOffline.emails as Array<{ subject?: string }>).map((e) => e.subject),
|
||||
).toContain(subject);
|
||||
|
||||
const mailboxesOffline = await (await call('/api/offline/mail?kind=mailboxes')).json();
|
||||
expect(mailboxesOffline.available).toBe(true);
|
||||
expect((mailboxesOffline.mailboxes as unknown[]).length).toBeGreaterThan(0);
|
||||
|
||||
// Status must be readable offline too - a user with no network still needs
|
||||
// to see what they have and be able to free the space.
|
||||
const statusOffline = await (await call('/api/offline/status')).json();
|
||||
expect(statusOffline.ok).toBe(true);
|
||||
expect(statusOffline.stats.bodies).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// A cycle attempted while offline must classify as Transport and must NOT
|
||||
// touch the data. "Offline is not an error."
|
||||
expect(
|
||||
['Transport', 'ServerTransient'].includes(String(afterCutBody.code)),
|
||||
`an offline cycle must classify as Transport/ServerTransient so the caller retries ` +
|
||||
`rather than treating the feature as broken; got code=${afterCutBody.code} ` +
|
||||
`status=${afterCut.status} body=${JSON.stringify(afterCutBody)}`,
|
||||
).toBe(true);
|
||||
const afterOfflineCycle = await (await call('/api/offline/status')).json();
|
||||
expect(
|
||||
afterOfflineCycle.stats.envelopes,
|
||||
'an offline cycle deleted data - a transport failure must never do that',
|
||||
).toBe(statusOffline.stats.envelopes);
|
||||
expect(afterOfflineCycle.resyncRequired).toBe(false);
|
||||
|
||||
// ── PURGE: the retention control has to actually free the space ────────
|
||||
const purge = await call('/api/offline/status', { method: 'DELETE' });
|
||||
expect(purge.status).toBe(200);
|
||||
const purged = await (await call('/api/offline/status')).json();
|
||||
expect(purged.synced).toBe(false);
|
||||
expect(purged.coveragePhase).toBe('never-run');
|
||||
} finally {
|
||||
server.kill();
|
||||
await proxy.stop();
|
||||
// Let the process release its WAL files before reading them.
|
||||
await new Promise((r) => setTimeout(r, 700));
|
||||
}
|
||||
|
||||
// ── the file on disk is genuinely encrypted ─────────────────────────────
|
||||
const accountId = `${alice.email}@127.0.0.1`;
|
||||
const dbPath = path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`);
|
||||
expect(fs.existsSync(dbPath), `no replica database at ${dbPath}`).toBe(true);
|
||||
|
||||
const onDisk = Buffer.concat(
|
||||
['', '-wal', '-shm']
|
||||
.map((suffix) => `${dbPath}${suffix}`)
|
||||
.filter((f) => fs.existsSync(f))
|
||||
.map((f) => fs.readFileSync(f)),
|
||||
);
|
||||
expect(onDisk.length).toBeGreaterThan(0);
|
||||
// `PRAGMA key` is a silent no-op on a non-SQLCipher binding - no error, a
|
||||
// working database, and the mail in cleartext - so every functional assertion
|
||||
// above would pass either way. These are the ones that catch it.
|
||||
expect(
|
||||
fs.readFileSync(dbPath).subarray(0, 15).toString('latin1'),
|
||||
'the replica file has a plain SQLite header - it is NOT encrypted',
|
||||
).not.toBe('SQLite format 3');
|
||||
expect(
|
||||
onDisk.includes(bodyPhrase),
|
||||
'the message body is recoverable from the raw database bytes - not encrypted',
|
||||
).toBe(false);
|
||||
expect(
|
||||
onDisk.includes(subject),
|
||||
'the subject is recoverable from the raw database bytes - not encrypted',
|
||||
).toBe(false);
|
||||
|
||||
fs.rmSync(storeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('wiring: the real standalone boot reaches the replica routes with a real safeStorage key', async () => {
|
||||
test.setTimeout(180_000);
|
||||
// A FRESH profile is load-bearing, not hygiene: the 401 asserted below is
|
||||
// "nobody is signed in", and a leftover jmap_stalwart_ctx cookie from any
|
||||
// previous run turns it into a 200.
|
||||
const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-wiring-'));
|
||||
const electronApp: ElectronApplication = await electron.launch({
|
||||
args: [projectRoot, `--user-data-dir=${userDataDir}`],
|
||||
env: {
|
||||
...process.env,
|
||||
JMAP_SERVER_URL: JMAP_URL,
|
||||
SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const appWindow: Page = await electronApp.firstWindow();
|
||||
await appWindow.waitForLoadState('domcontentloaded');
|
||||
await appWindow.locator('#username').waitFor({ state: 'visible', timeout: 90_000 });
|
||||
|
||||
const encryptionAvailable = await electronApp.evaluate(({ safeStorage }) =>
|
||||
safeStorage.isEncryptionAvailable(),
|
||||
);
|
||||
expect(
|
||||
encryptionAvailable,
|
||||
'safeStorage reports no encryption available, so main.ts correctly disabled the ' +
|
||||
'feature - this assertion cannot pass here',
|
||||
).toBe(true);
|
||||
|
||||
// 401 = the gate opened, the native binding loaded from the REAL standalone
|
||||
// artifact, and the fd-3 key channel is present; it refuses only because
|
||||
// nobody is signed in.
|
||||
// 404 => VNCMAIL_DESKTOP_STORE_DIR was never set
|
||||
// 503 => the native binding or the key channel is missing from the real
|
||||
// build - the class of failure only a real build reveals
|
||||
for (const route of [
|
||||
'/api/offline/status',
|
||||
'/api/offline/mail?kind=mailboxes',
|
||||
'/api/offline/sync',
|
||||
]) {
|
||||
const probe = await appWindow.evaluate(async (url) => {
|
||||
const response = await fetch(url, {
|
||||
method: url.endsWith('/sync') ? 'POST' : 'GET',
|
||||
});
|
||||
return { status: response.status, body: (await response.text()).slice(0, 300) };
|
||||
}, route);
|
||||
expect(
|
||||
probe.status,
|
||||
`${route}: expected 401 (reachable, unauthenticated) but got ${probe.status}: ${probe.body}`,
|
||||
).toBe(401);
|
||||
}
|
||||
} finally {
|
||||
await electronApp.close();
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Global setup for playwright.integration-electron.config.ts - a narrower
|
||||
* variant of ./global-setup.ts.
|
||||
*
|
||||
* The Electron suite (11-electron-notification.spec.ts) boots its OWN
|
||||
* standalone Next.js server via electron/main.ts, so unlike the main
|
||||
* integration config it never talks to the docker-compose `webmail`
|
||||
* container on :3000 at all - only to `stalwart` (JMAP + SMTP). Bringing up
|
||||
* `webmail` too would be pointless work, and on a host where something else
|
||||
* already owns port 3000 (this repo doesn't own that port - any other
|
||||
* project's dev server can be sitting on it) it would fail outright for a
|
||||
* container this suite never uses. `docker compose up <service>` scopes the
|
||||
* bring-up to just `stalwart`.
|
||||
*
|
||||
* Set IT_NO_DOCKER=1 to skip container management entirely (useful when the
|
||||
* stack is already running).
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { existsSync, copyFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { JMAP_URL, ACCOUNTS, ACCOUNT_PASSWORD } from './helpers/config';
|
||||
|
||||
const INTEGRATION_DIR = path.resolve(__dirname, '..');
|
||||
const COMPOSE_FILE = path.join(INTEGRATION_DIR, 'docker-compose.yml');
|
||||
const ENV_FILE = path.join(INTEGRATION_DIR, '.env');
|
||||
const STALWART_CLI_BIN = path.join(INTEGRATION_DIR, 'stalwart', 'stalwart-cli');
|
||||
|
||||
function run(cmd: string, args: string[]): void {
|
||||
execFileSync(cmd, args, { cwd: INTEGRATION_DIR, stdio: 'inherit' });
|
||||
}
|
||||
|
||||
async function waitForStalwart(timeoutMs = 240000): Promise<void> {
|
||||
const url = `${JMAP_URL}/jmap/session`;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const auth = 'Basic ' + Buffer.from(`${ACCOUNTS.alice.email}:${ACCOUNT_PASSWORD}`).toString('base64');
|
||||
for (;;) {
|
||||
try {
|
||||
const res = await fetch(url, { headers: { Authorization: auth } });
|
||||
if (res.ok) return;
|
||||
} catch {
|
||||
/* not up yet */
|
||||
}
|
||||
if (Date.now() > deadline) throw new Error(`Timed out waiting for Stalwart JMAP at ${url}`);
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
}
|
||||
|
||||
export default async function globalSetup(): Promise<void> {
|
||||
if (process.env.IT_NO_DOCKER === '1') {
|
||||
console.log('[global-setup-electron] IT_NO_DOCKER=1 - skipping docker compose management');
|
||||
} else {
|
||||
// stalwart/prepare-stalwart-cli.sh fetches a LINUX binary (it's COPYed
|
||||
// into the Stalwart container by integration/stalwart/Dockerfile - never
|
||||
// meant to run on the host at all) but ends by executing it as its own
|
||||
// sanity check, which only works when the host itself is Linux. On a
|
||||
// macOS host that self-check fails outright ("cannot execute binary
|
||||
// file") even though the download+extract already succeeded and the
|
||||
// file the Dockerfile needs is perfectly fine on disk. Skipping the
|
||||
// script once the binary already exists sidesteps that host/target
|
||||
// mismatch without touching the shared script (used by the main
|
||||
// integration config too, on hosts where it does work).
|
||||
if (existsSync(STALWART_CLI_BIN)) {
|
||||
console.log('[global-setup-electron] stalwart-cli already present, skipping fetch');
|
||||
} else {
|
||||
console.log('[global-setup-electron] fetching stalwart-cli');
|
||||
run('bash', [path.join(INTEGRATION_DIR, 'stalwart', 'prepare-stalwart-cli.sh')]);
|
||||
}
|
||||
|
||||
if (!existsSync(ENV_FILE)) {
|
||||
console.log('[global-setup-electron] creating integration/.env from .env.example');
|
||||
copyFileSync(path.join(INTEGRATION_DIR, '.env.example'), ENV_FILE);
|
||||
}
|
||||
|
||||
console.log('[global-setup-electron] docker compose up -d --build --wait stalwart');
|
||||
run('docker', [
|
||||
'compose', '-f', COMPOSE_FILE, '--env-file', ENV_FILE,
|
||||
'up', '-d', '--build', '--wait', '--wait-timeout', '300', 'stalwart',
|
||||
]);
|
||||
}
|
||||
|
||||
console.log('[global-setup-electron] waiting for Stalwart JMAP');
|
||||
await waitForStalwart();
|
||||
|
||||
console.log('[global-setup-electron] stack ready');
|
||||
}
|
||||
@@ -22,6 +22,14 @@ interface SendOptions {
|
||||
subject: string;
|
||||
/** Plain-text body. */
|
||||
body: string;
|
||||
/**
|
||||
* Optional HTML alternative, sent as multipart/alternative alongside `body`.
|
||||
*
|
||||
* Added for 13-electron-offline-replica.spec.ts, which has to prove the offline
|
||||
* replica stores a real HTML body and not just the plain-text excerpt the search
|
||||
* index keeps - so the message needs a genuine distinct text/html part.
|
||||
*/
|
||||
html?: string;
|
||||
/** Extra headers (e.g. custom Message-ID / In-Reply-To for threading). */
|
||||
headers?: Record<string, string>;
|
||||
/** Optional single attachment (sent as multipart/mixed, base64). */
|
||||
@@ -158,6 +166,22 @@ export async function sendMail(opts: SendOptions): Promise<void> {
|
||||
b64,
|
||||
`--${boundary}--`,
|
||||
].join('\r\n');
|
||||
} else if (opts.html) {
|
||||
const boundary = 'italt_boundary_0001';
|
||||
headers['MIME-Version'] = '1.0';
|
||||
headers['Content-Type'] = `multipart/alternative; boundary="${boundary}"`;
|
||||
// text first, html second: multipart/alternative is least-to-most preferred.
|
||||
mime = [
|
||||
`--${boundary}`,
|
||||
'Content-Type: text/plain; charset=utf-8',
|
||||
'',
|
||||
crlf(opts.body),
|
||||
`--${boundary}`,
|
||||
'Content-Type: text/html; charset=utf-8',
|
||||
'',
|
||||
crlf(opts.html),
|
||||
`--${boundary}--`,
|
||||
].join('\r\n');
|
||||
} else {
|
||||
headers['Content-Type'] = 'text/plain; charset=utf-8';
|
||||
mime = crlf(opts.body);
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
// The two-part fallback gate.
|
||||
//
|
||||
// `lib/jmap/client.ts`'s read methods swallow their own errors and return
|
||||
// plausible success, so a "looks empty" result is NOT evidence of a network
|
||||
// failure - it is also what a genuinely empty folder returns, and
|
||||
// `getMailboxes()` fabricates a synthetic Inbox rather than throwing. Falling back
|
||||
// on the shape alone would serve stale replica rows over a folder the user had
|
||||
// just emptied. So the gate is: suspicious result AND a `fetch` rejection recorded
|
||||
// during that exact call.
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { IJMAPClient } from '@/lib/jmap/client-interface';
|
||||
import type { Email, Mailbox } from '@/lib/jmap/types';
|
||||
import { noteTransportFailure, resetTransportHealth } from '@/lib/jmap/transport-health';
|
||||
|
||||
const readOfflineMailboxes = vi.fn();
|
||||
const readOfflineList = vi.fn();
|
||||
const readOfflineMessage = vi.fn();
|
||||
const isReplicaUnavailable = vi.fn(() => false);
|
||||
|
||||
vi.mock('@/lib/offline-replica-client', () => ({
|
||||
readOfflineMailboxes: (...a: unknown[]) => readOfflineMailboxes(...a),
|
||||
readOfflineList: (...a: unknown[]) => readOfflineList(...a),
|
||||
readOfflineMessage: (...a: unknown[]) => readOfflineMessage(...a),
|
||||
isReplicaUnavailable: () => isReplicaUnavailable(),
|
||||
}));
|
||||
|
||||
vi.mock('@/stores/account-store', () => ({
|
||||
useAccountStore: {
|
||||
getState: () => ({
|
||||
accounts: [{ id: 'alice@mail.example.org', cookieSlot: 3, serverIdentifiers: [] }],
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const { withOfflineFallback } = await import('@/lib/offline-fallback-client');
|
||||
|
||||
function replicaEmail(id: string): Email {
|
||||
return {
|
||||
id, threadId: 't', mailboxIds: { inbox: true }, keywords: {}, size: 1,
|
||||
receivedAt: '2026-08-01T00:00:00.000Z', hasAttachment: false,
|
||||
htmlBody: [{ partId: '1', blobId: 'b', size: 1, type: 'text/html' }],
|
||||
bodyValues: { '1': { value: '<p>from the replica</p>' } },
|
||||
};
|
||||
}
|
||||
|
||||
interface Stub extends Partial<IJMAPClient> {
|
||||
getEmail: IJMAPClient['getEmail'];
|
||||
getEmails: IJMAPClient['getEmails'];
|
||||
getMailboxes: IJMAPClient['getMailboxes'];
|
||||
getAllMailboxes: IJMAPClient['getAllMailboxes'];
|
||||
}
|
||||
|
||||
/** Reproduces the client's real error-swallowing shapes. */
|
||||
function stubClient(overrides: Partial<Stub> = {}): IJMAPClient {
|
||||
const stub = {
|
||||
getUsername: () => 'alice',
|
||||
getServerUrl: () => 'https://mail.example.org',
|
||||
getAccountId: () => 'primary',
|
||||
getEmail: async () => null,
|
||||
getEmails: async () => ({ emails: [] as Email[], hasMore: false, total: 0 }),
|
||||
getMailboxes: async () => ([
|
||||
// The exact placeholder client.ts fabricates on failure.
|
||||
{ id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0,
|
||||
unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true,
|
||||
myRights: {} } as unknown as Mailbox,
|
||||
]),
|
||||
getAllMailboxes: async () => ([
|
||||
{ id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0,
|
||||
unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true,
|
||||
myRights: {} } as unknown as Mailbox,
|
||||
]),
|
||||
...overrides,
|
||||
};
|
||||
return stub as unknown as IJMAPClient;
|
||||
}
|
||||
|
||||
describe('withOfflineFallback', () => {
|
||||
beforeEach(() => {
|
||||
resetTransportHealth();
|
||||
vi.clearAllMocks();
|
||||
isReplicaUnavailable.mockReturnValue(false);
|
||||
});
|
||||
|
||||
it('does NOT consult the replica when the server answered "empty"', async () => {
|
||||
// The whole point. An empty folder must render empty, not as whatever the
|
||||
// replica last held.
|
||||
const client = withOfflineFallback(stubClient());
|
||||
const result = await client.getEmails('inbox');
|
||||
expect(result.emails).toEqual([]);
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
|
||||
expect(await client.getEmail('e1')).toBeNull();
|
||||
expect(readOfflineMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('consults the replica when a transport failure happened DURING the call', async () => {
|
||||
readOfflineList.mockResolvedValue({
|
||||
emails: [replicaEmail('e1')], total: 1, hasMore: false,
|
||||
});
|
||||
const client = withOfflineFallback(
|
||||
stubClient({
|
||||
getEmails: async () => {
|
||||
// What authenticatedFetch does when `fetch` rejects.
|
||||
noteTransportFailure();
|
||||
return { emails: [], hasMore: false, total: 0 };
|
||||
},
|
||||
}),
|
||||
);
|
||||
const result = await client.getEmails('inbox', undefined, 25, 0);
|
||||
expect(result.emails.map((e) => e.id)).toEqual(['e1']);
|
||||
expect(result.total).toBe(1);
|
||||
// And it asks for the right slot, so a multi-account shell reads the right file.
|
||||
expect(readOfflineList).toHaveBeenCalledWith('inbox', { limit: 25, offset: 0, slot: 3 });
|
||||
});
|
||||
|
||||
it('ignores a stale transport failure from BEFORE the call', async () => {
|
||||
// The counter is sampled per call precisely so an old failure cannot make a
|
||||
// later successful-but-empty read look offline.
|
||||
noteTransportFailure();
|
||||
const client = withOfflineFallback(stubClient());
|
||||
await client.getEmails('inbox');
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('serves a full message from the replica, but refuses an envelope-only hit', async () => {
|
||||
// An envelope with no bodyValues would render blank AND leave the viewer's
|
||||
// isBodyLoading gate stuck on its skeleton, which is worse than saying the
|
||||
// message is unavailable.
|
||||
readOfflineMessage.mockResolvedValue({ email: replicaEmail('e1'), hasBody: true });
|
||||
const client = withOfflineFallback(
|
||||
stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }),
|
||||
);
|
||||
const email = await client.getEmail('e1');
|
||||
expect(email?.bodyValues?.['1'].value).toContain('from the replica');
|
||||
|
||||
resetTransportHealth();
|
||||
readOfflineMessage.mockResolvedValue({ email: replicaEmail('e2'), hasBody: false });
|
||||
const client2 = withOfflineFallback(
|
||||
stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }),
|
||||
);
|
||||
expect(await client2.getEmail('e2')).toBeNull();
|
||||
});
|
||||
|
||||
it('recognises the synthetic Inbox placeholder and replaces it', async () => {
|
||||
readOfflineMailboxes.mockResolvedValue([
|
||||
{ id: 'mb1', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 9, unreadEmails: 2,
|
||||
totalThreads: 9, unreadThreads: 2, isSubscribed: true, myRights: {} } as unknown as Mailbox,
|
||||
]);
|
||||
const client = withOfflineFallback(
|
||||
stubClient({
|
||||
getAllMailboxes: async () => {
|
||||
noteTransportFailure();
|
||||
return [
|
||||
{ id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0,
|
||||
unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true,
|
||||
myRights: {} } as unknown as Mailbox,
|
||||
];
|
||||
},
|
||||
}),
|
||||
);
|
||||
const mailboxes = await client.getAllMailboxes();
|
||||
expect(mailboxes.map((m) => m.id)).toEqual(['mb1']);
|
||||
});
|
||||
|
||||
it('keeps a REAL single-mailbox server result even after a transport failure', async () => {
|
||||
// A genuine server that happens to return one inbox has a real id and real
|
||||
// counts; only the exact placeholder shape may be replaced.
|
||||
const real = {
|
||||
id: 'real-inbox-id', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 12,
|
||||
unreadEmails: 1, totalThreads: 12, unreadThreads: 1, isSubscribed: true, myRights: {},
|
||||
} as unknown as Mailbox;
|
||||
const client = withOfflineFallback(
|
||||
stubClient({ getAllMailboxes: async () => { noteTransportFailure(); return [real]; } }),
|
||||
);
|
||||
expect((await client.getAllMailboxes())[0].id).toBe('real-inbox-id');
|
||||
expect(readOfflineMailboxes).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('never answers a read scoped to a delegated account', async () => {
|
||||
// v1 replicates the PRIMARY mail account only, so the replica has no rows for
|
||||
// a shared account and answering "empty" would be worse than the client's own.
|
||||
const client = withOfflineFallback(
|
||||
stubClient({
|
||||
getEmails: async () => { noteTransportFailure(); return { emails: [], hasMore: false, total: 0 }; },
|
||||
}),
|
||||
);
|
||||
await client.getEmails('inbox', 'someone-elses-account');
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('never answers a keyword- or category-filtered query', async () => {
|
||||
// Those are server-side queries the replica does not reproduce. Serving an
|
||||
// unfiltered page in their place would silently show the wrong set.
|
||||
const failing = async () => { noteTransportFailure(); return { emails: [], hasMore: false, total: 0 }; };
|
||||
const c1 = withOfflineFallback(stubClient({ getEmails: failing }));
|
||||
await c1.getEmails('inbox', undefined, 25, 0, '$flagged');
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
|
||||
resetTransportHealth();
|
||||
const c2 = withOfflineFallback(stubClient({ getEmails: failing }));
|
||||
await c2.getEmails('inbox', undefined, 25, 0, undefined, true, { from: 'x' });
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('stops asking once the replica reports itself absent', async () => {
|
||||
isReplicaUnavailable.mockReturnValue(true);
|
||||
const client = withOfflineFallback(
|
||||
stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }),
|
||||
);
|
||||
expect(await client.getEmail('e1')).toBeNull();
|
||||
expect(readOfflineMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is idempotent, so re-wrapping a client does not stack fallbacks', async () => {
|
||||
readOfflineMessage.mockResolvedValue({ email: replicaEmail('e1'), hasBody: true });
|
||||
const base = stubClient({ getEmail: async () => { noteTransportFailure(); return null; } });
|
||||
const once = withOfflineFallback(base);
|
||||
const twice = withOfflineFallback(once);
|
||||
expect(twice).toBe(once);
|
||||
await twice.getEmail('e1');
|
||||
expect(readOfflineMessage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,300 @@
|
||||
import { readFile, readdir, stat } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { logger } from '@/lib/logger';
|
||||
import { ALL_PERMISSIONS, MAX_PLUGIN_SIZE } from '@/lib/plugin-types';
|
||||
import { auditLog } from './audit';
|
||||
import { configManager } from './config-manager';
|
||||
import { isConfigReadOnly } from './paths';
|
||||
import {
|
||||
getPluginRegistry,
|
||||
savePlugin,
|
||||
updatePluginMeta,
|
||||
type ServerPlugin,
|
||||
} from './plugin-registry';
|
||||
import type { FeatureGates } from './types';
|
||||
|
||||
/**
|
||||
* First-party ("bundled") plugin installation.
|
||||
*
|
||||
* The plugin registry (`<CONFIG_DIR>/plugins/`) is the host's admin channel:
|
||||
* bundles served out of it are Ed25519-signed on the way out by
|
||||
* `app/api/admin/plugins/[id]/bundle`, and `/api/plugins` is what makes a
|
||||
* plugin `managed` on the client - which is what `resolvePluginTier` requires
|
||||
* before it will grant the privileged (same-origin) tier.
|
||||
*
|
||||
* This module installs the plugins this fork ships with THROUGH that same
|
||||
* channel, so nothing about the signing / approval / consent chain is
|
||||
* bypassed or relaxed: the operator's own server performs the install that an
|
||||
* operator would otherwise perform by uploading the ZIP in /admin.
|
||||
*
|
||||
* Input is the staging directory produced by `scripts/build-plugins.mjs`:
|
||||
*
|
||||
* vnc/plugins/build/<id>/manifest.json
|
||||
* vnc/plugins/build/<id>/<entrypoint>
|
||||
*
|
||||
* Nothing here is trusted blindly - the manifest is validated the same way the
|
||||
* admin upload route validates one, and an unknown permission or a bad id is a
|
||||
* refusal, not a warning.
|
||||
*/
|
||||
|
||||
/** Where the staged bundles live, relative to cwd (override for odd layouts). */
|
||||
function getBundledPluginsDir(): string {
|
||||
return (
|
||||
process.env.BUNDLED_PLUGINS_DIR ||
|
||||
path.join(process.cwd(), 'vnc', 'plugins', 'build')
|
||||
);
|
||||
}
|
||||
|
||||
interface FirstPartyPlugin {
|
||||
id: string;
|
||||
/**
|
||||
* Feature gate that decides whether this plugin is installed and served.
|
||||
* Turning the gate off in the admin policy disables the plugin (and stops
|
||||
* it being re-installed on the next boot) - that, not the Delete button, is
|
||||
* the way to remove a bundled plugin, since a delete would be undone by the
|
||||
* next restart.
|
||||
*/
|
||||
gate: keyof FeatureGates;
|
||||
/**
|
||||
* Force-enable for every user. Required for a bundled plugin to be reachable
|
||||
* at all under the default policy: `pluginsEnabled` defaults to false, which
|
||||
* hides the user-facing Settings > Plugins tab, so there would be no way for
|
||||
* a user to switch it on by hand.
|
||||
*/
|
||||
forceEnable: boolean;
|
||||
}
|
||||
|
||||
const FIRST_PARTY_PLUGINS: FirstPartyPlugin[] = [
|
||||
// The audited S/MIME implementation (vnc/plugins/smime). It IS the S/MIME
|
||||
// feature - the former in-host native pipeline is gone - so the long-standing
|
||||
// `smimeEnabled` policy gate now controls this plugin.
|
||||
{ id: 'smime', gate: 'smimeEnabled', forceEnable: true },
|
||||
];
|
||||
|
||||
const ID_RE = /^[a-z0-9][a-z0-9-]*[a-z0-9]$/;
|
||||
const VALID_TYPES = new Set(['ui-extension', 'sidebar-app', 'hook']);
|
||||
|
||||
function asString(v: unknown, fallback = ''): string {
|
||||
return typeof v === 'string' ? v : fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a staged manifest and turn it into a registry entry. Returns a list
|
||||
* of errors instead of throwing so one bad bundle can't take down startup.
|
||||
*/
|
||||
function toServerPlugin(
|
||||
manifest: Record<string, unknown>,
|
||||
code: string,
|
||||
opts: { enabled: boolean; forceEnabled: boolean; installedAt: string },
|
||||
): { plugin: ServerPlugin } | { errors: string[] } {
|
||||
const errors: string[] = [];
|
||||
const id = asString(manifest.id);
|
||||
if (!ID_RE.test(id)) errors.push(`invalid id ${JSON.stringify(manifest.id)}`);
|
||||
if (!asString(manifest.name)) errors.push('missing "name"');
|
||||
if (!asString(manifest.version)) errors.push('missing "version"');
|
||||
if (!asString(manifest.author)) errors.push('missing "author"');
|
||||
const entrypoint = asString(manifest.entrypoint);
|
||||
if (!entrypoint) errors.push('missing "entrypoint"');
|
||||
if (!VALID_TYPES.has(asString(manifest.type))) {
|
||||
errors.push(`invalid type ${JSON.stringify(manifest.type)}`);
|
||||
}
|
||||
|
||||
const permissions = Array.isArray(manifest.permissions)
|
||||
? manifest.permissions.filter((p): p is string => typeof p === 'string')
|
||||
: [];
|
||||
const known = new Set<string>(ALL_PERMISSIONS as readonly string[]);
|
||||
const unknownPerms = permissions.filter(p => !known.has(p));
|
||||
if (unknownPerms.length > 0) {
|
||||
errors.push(`unknown permissions: ${unknownPerms.join(', ')}`);
|
||||
}
|
||||
|
||||
const size = Buffer.byteLength(code, 'utf-8');
|
||||
if (size > MAX_PLUGIN_SIZE) {
|
||||
errors.push(`bundle is ${size} bytes, over the ${MAX_PLUGIN_SIZE} byte limit`);
|
||||
}
|
||||
|
||||
if (errors.length > 0) return { errors };
|
||||
|
||||
return {
|
||||
plugin: {
|
||||
id,
|
||||
name: asString(manifest.name),
|
||||
version: asString(manifest.version),
|
||||
author: asString(manifest.author),
|
||||
description: asString(manifest.description),
|
||||
type: asString(manifest.type),
|
||||
// Only 'privileged' is meaningful; anything else falls through to the
|
||||
// default untrusted tier. Same narrowing the admin upload route applies.
|
||||
...(manifest.tier === 'privileged' ? { tier: 'privileged' } : {}),
|
||||
permissions,
|
||||
entrypoint,
|
||||
enabled: opts.enabled,
|
||||
forceEnabled: opts.forceEnabled,
|
||||
...(manifest.configSchema && typeof manifest.configSchema === 'object'
|
||||
? { configSchema: manifest.configSchema as ServerPlugin['configSchema'] }
|
||||
: {}),
|
||||
...(manifest.settingsSchema && typeof manifest.settingsSchema === 'object'
|
||||
? { settingsSchema: manifest.settingsSchema as ServerPlugin['settingsSchema'] }
|
||||
: {}),
|
||||
...(manifest.locales && typeof manifest.locales === 'object'
|
||||
? { locales: manifest.locales as ServerPlugin['locales'] }
|
||||
: {}),
|
||||
installedAt: opts.installedAt,
|
||||
updatedAt: opts.installedAt,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readStaged(dir: string, id: string): Promise<
|
||||
{ manifest: Record<string, unknown>; code: string } | null
|
||||
> {
|
||||
const manifestPath = path.join(dir, id, 'manifest.json');
|
||||
if (!existsSync(manifestPath)) return null;
|
||||
|
||||
let manifest: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
||||
if (typeof parsed !== 'object' || parsed === null) throw new Error('not an object');
|
||||
manifest = parsed as Record<string, unknown>;
|
||||
} catch (err) {
|
||||
logger.error(`[bundled-plugins] ${id}: unreadable manifest`, {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const entrypoint = asString(manifest.entrypoint, 'index.js');
|
||||
if (entrypoint.includes('/') || entrypoint.includes('\\')) {
|
||||
logger.error(`[bundled-plugins] ${id}: entrypoint must be a bare filename`);
|
||||
return null;
|
||||
}
|
||||
const codePath = path.join(dir, id, entrypoint);
|
||||
try {
|
||||
return { manifest, code: await readFile(codePath, 'utf-8') };
|
||||
} catch (err) {
|
||||
logger.error(`[bundled-plugins] ${id}: cannot read bundle ${entrypoint}`, {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Install / update / disable the bundled first-party plugins. Idempotent: a
|
||||
* boot where nothing changed writes nothing.
|
||||
*
|
||||
* Never throws - a failure here must not stop the server from starting, it
|
||||
* just means the plugin isn't there (and says so in the log).
|
||||
*/
|
||||
export async function seedBundledPlugins(): Promise<void> {
|
||||
try {
|
||||
const dir = getBundledPluginsDir();
|
||||
const staged = existsSync(dir);
|
||||
|
||||
await configManager.ensureLoaded();
|
||||
const features = configManager.getPolicy().features;
|
||||
const registry = await getPluginRegistry();
|
||||
|
||||
for (const spec of FIRST_PARTY_PLUGINS) {
|
||||
const gateOn = features[spec.gate] !== false;
|
||||
const existing = registry.plugins.find(p => p.id === spec.id);
|
||||
|
||||
if (!gateOn) {
|
||||
// Policy says off. Stop serving it (the client's own sync then treats
|
||||
// it as removed and cleans it up) but leave the bundle on disk so
|
||||
// flipping the gate back on is instant.
|
||||
if (existing && (existing.enabled || existing.forceEnabled)) {
|
||||
if (isConfigReadOnly()) {
|
||||
logger.warn(
|
||||
`[bundled-plugins] ${spec.id}: policy "${spec.gate}" is off but the ` +
|
||||
'config dir is read-only, so it stays enabled',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
await updatePluginMeta(spec.id, { enabled: false, forceEnabled: false });
|
||||
logger.info(`[bundled-plugins] ${spec.id} disabled ("${spec.gate}" is off in policy)`);
|
||||
await auditLog('plugin.bundled.disable', { id: spec.id, gate: spec.gate }, 'system');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!staged) continue;
|
||||
|
||||
const read = await readStaged(dir, spec.id);
|
||||
if (!read) {
|
||||
logger.warn(
|
||||
`[bundled-plugins] ${spec.id}: not staged in ${dir} - ` +
|
||||
'run "npm run build:plugins" (the container and standalone builds do this for you)',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const bundleHash = createHash('sha256').update(read.code).digest('hex');
|
||||
const version = asString(read.manifest.version);
|
||||
const unchanged =
|
||||
existing !== undefined &&
|
||||
existing.version === version &&
|
||||
existing.bundleHash === bundleHash &&
|
||||
existing.enabled === true &&
|
||||
existing.forceEnabled === spec.forceEnable;
|
||||
if (unchanged) {
|
||||
logger.debug(`[bundled-plugins] ${spec.id} v${version} already installed`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isConfigReadOnly()) {
|
||||
logger.warn(
|
||||
`[bundled-plugins] ${spec.id} v${version} cannot be installed: the admin ` +
|
||||
'config dir is read-only. Remount it read-write (or unset ' +
|
||||
'ADMIN_CONFIG_READONLY) once, so the plugin registry can be written.',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const built = toServerPlugin(read.manifest, read.code, {
|
||||
enabled: true,
|
||||
forceEnabled: spec.forceEnable,
|
||||
installedAt: existing?.installedAt ?? new Date().toISOString(),
|
||||
});
|
||||
if ('errors' in built) {
|
||||
logger.error(`[bundled-plugins] ${spec.id}: manifest rejected`, {
|
||||
errors: built.errors.join('; '),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
await savePlugin(built.plugin, read.code);
|
||||
const action = existing ? 'update' : 'install';
|
||||
logger.info(
|
||||
`[bundled-plugins] ${existing ? 'updated' : 'installed'} ${spec.id} v${version} ` +
|
||||
`(tier=${built.plugin.tier ?? 'untrusted'}, forceEnabled=${spec.forceEnable})`,
|
||||
);
|
||||
await auditLog(
|
||||
`plugin.bundled.${action}`,
|
||||
{ id: spec.id, version, bundleHash, tier: built.plugin.tier ?? 'untrusted' },
|
||||
'system',
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error('[bundled-plugins] seeding failed', {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Exposed for diagnostics / tests. */
|
||||
export async function listStagedBundledPlugins(): Promise<string[]> {
|
||||
const dir = getBundledPluginsDir();
|
||||
if (!existsSync(dir)) return [];
|
||||
const names = await readdir(dir);
|
||||
const out: string[] = [];
|
||||
for (const name of names) {
|
||||
if (name.startsWith('.')) continue;
|
||||
try {
|
||||
if ((await stat(path.join(dir, name))).isDirectory()) out.push(name);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -193,10 +193,22 @@ async function loadDevPlugin(pluginDir: string): Promise<DevPluginEntry | null>
|
||||
author: asString(manifest.author),
|
||||
description: asString(manifest.description),
|
||||
type: asString(manifest.type, 'hook'),
|
||||
// Requested execution tier. Dropped here previously, which silently pinned
|
||||
// every dev-loaded plugin to the untrusted (null-origin) tier - so a
|
||||
// privileged plugin such as S/MIME could never be exercised from
|
||||
// PLUGIN_DEV_DIR. Only 'privileged' is meaningful (same narrowing as the
|
||||
// admin upload route); the tier is still *granted* only by
|
||||
// resolvePluginTier, which additionally requires managed + consent.
|
||||
...(manifest.tier === 'privileged' ? { tier: 'privileged' } : {}),
|
||||
permissions,
|
||||
entrypoint,
|
||||
enabled: true,
|
||||
forceEnabled: false,
|
||||
// Manifest i18n tables - also previously dropped, so api.i18n.t() fell back
|
||||
// to raw keys for dev-loaded plugins.
|
||||
...(manifest.locales && typeof manifest.locales === 'object'
|
||||
? { locales: manifest.locales as ServerPlugin['locales'] }
|
||||
: {}),
|
||||
...(manifest.configSchema && typeof manifest.configSchema === 'object'
|
||||
? { configSchema: manifest.configSchema as ServerPlugin['configSchema'] }
|
||||
: {}),
|
||||
|
||||
+1
-1
@@ -106,7 +106,7 @@ export interface ThemePolicy {
|
||||
export const DEFAULT_THEME_POLICY: ThemePolicy = {
|
||||
disabledBuiltinThemes: [],
|
||||
disabledThemes: [],
|
||||
defaultThemeId: 'builtin-vnclagoon',
|
||||
defaultThemeId: 'builtin-src',
|
||||
};
|
||||
|
||||
export interface SettingsPolicy {
|
||||
|
||||
+230
-4
@@ -1014,6 +1014,11 @@ body[data-theme-skin="builtin-vnclagoon"] .rounded-2xl.border.bg-background\\/80
|
||||
// near-black stone-grey neutral. Dark variant brightens the red (#EF4444) on a
|
||||
// warm near-black. Info stays blue so it never collides with the red accent.
|
||||
const srcCSS = `
|
||||
@font-face { font-family: 'DM Sans'; font-style: normal; font-weight: 400; font-display: swap; src: url('/fonts/dmsans-400.woff2') format('woff2'); }
|
||||
@font-face { font-family: 'DM Sans'; font-style: normal; font-weight: 500; font-display: swap; src: url('/fonts/dmsans-500.woff2') format('woff2'); }
|
||||
@font-face { font-family: 'DM Sans'; font-style: normal; font-weight: 700; font-display: swap; src: url('/fonts/dmsans-700.woff2') format('woff2'); }
|
||||
@font-face { font-family: 'Syne'; font-style: normal; font-weight: 700; font-display: swap; src: url('/fonts/syne-700.woff2') format('woff2'); }
|
||||
@font-face { font-family: 'Syne'; font-style: normal; font-weight: 800; font-display: swap; src: url('/fonts/syne-800.woff2') format('woff2'); }
|
||||
:root {
|
||||
--color-border: #e7e5e4;
|
||||
--color-input: #e7e5e4;
|
||||
@@ -1095,6 +1100,225 @@ const srcCSS = `
|
||||
--color-chart-5: #a78bfa;
|
||||
}`;
|
||||
|
||||
// MD3 component overrides for the SRC theme — shape scale, filled buttons,
|
||||
// text fields, cards, dialogs, state layers, switches, login card treatment.
|
||||
// All scoped under the skin body attribute so they detach cleanly on switch-off.
|
||||
const srcSkin = `
|
||||
body[data-theme-skin="builtin-src"] {
|
||||
font-family: "DM Sans", system-ui, -apple-system, "Segoe UI", sans-serif;
|
||||
}
|
||||
body[data-theme-skin="builtin-src"] h1,
|
||||
body[data-theme-skin="builtin-src"] h2,
|
||||
body[data-theme-skin="builtin-src"] h3 {
|
||||
font-family: "Syne", "DM Sans", sans-serif;
|
||||
font-weight: 700;
|
||||
letter-spacing: -0.01em;
|
||||
}
|
||||
|
||||
/* ── MD3 Shape scale ─────────────────────────────────────────── */
|
||||
/* Remap Tailwind rounded-* to M3 shape tokens. rounded-full (pill */
|
||||
/* avatars, badges, toggles) is intentionally left untouched. */
|
||||
body[data-theme-skin="builtin-src"] .rounded-sm { border-radius: 4px !important; }
|
||||
body[data-theme-skin="builtin-src"] .rounded { border-radius: 4px !important; }
|
||||
body[data-theme-skin="builtin-src"] .rounded-md { border-radius: 8px !important; }
|
||||
body[data-theme-skin="builtin-src"] .rounded-lg { border-radius: 12px !important; }
|
||||
body[data-theme-skin="builtin-src"] .rounded-xl { border-radius: 16px !important; }
|
||||
body[data-theme-skin="builtin-src"] .rounded-2xl { border-radius: 28px !important; }
|
||||
body[data-theme-skin="builtin-src"] .rounded-3xl { border-radius: 28px !important; }
|
||||
|
||||
/* ── MD3 Buttons: full shape (20 dp) ─────────────────────────── */
|
||||
/* All button variants (filled, tonal, outlined, text) use 20 dp */
|
||||
/* corners per M3. Circle icon buttons (.rounded-full) are skipped; */
|
||||
/* switches ([role="switch"]) are handled separately. */
|
||||
body[data-theme-skin="builtin-src"] button:not(.rounded-full):not([role="switch"]) {
|
||||
border-radius: 20px !important;
|
||||
}
|
||||
|
||||
/* MD3 filled button — primary surface, M3 label-large, state layers */
|
||||
body[data-theme-skin="builtin-src"] .bg-primary.text-primary-foreground {
|
||||
border-radius: 20px !important;
|
||||
padding-inline: 24px !important;
|
||||
min-height: 40px !important;
|
||||
font-weight: 500 !important;
|
||||
letter-spacing: 0.0063em !important;
|
||||
border: none !important;
|
||||
box-shadow: none !important;
|
||||
transition: box-shadow 200ms ease, filter 200ms ease;
|
||||
}
|
||||
/* hover: M3 elevation 1 + 8 % on-primary state layer */
|
||||
body[data-theme-skin="builtin-src"] .bg-primary.text-primary-foreground:hover {
|
||||
box-shadow:
|
||||
0 1px 2px rgba(0, 0, 0, 0.30),
|
||||
0 2px 6px 2px rgba(0, 0, 0, 0.15) !important;
|
||||
filter: brightness(1.06);
|
||||
}
|
||||
/* focus: +12 % tint + M3 focus ring */
|
||||
body[data-theme-skin="builtin-src"] .bg-primary.text-primary-foreground:focus-visible {
|
||||
filter: brightness(1.10) !important;
|
||||
outline: 3px solid var(--color-ring) !important;
|
||||
outline-offset: 2px !important;
|
||||
}
|
||||
/* pressed: +12 % darker, no shadow */
|
||||
body[data-theme-skin="builtin-src"] .bg-primary.text-primary-foreground:active {
|
||||
box-shadow: none !important;
|
||||
filter: brightness(0.94) !important;
|
||||
}
|
||||
|
||||
/* ── MD3 Text fields: outlined style, extra-small (4 dp) ─────── */
|
||||
body[data-theme-skin="builtin-src"] input:not([type="checkbox"]):not([type="radio"]):not([type="range"]),
|
||||
body[data-theme-skin="builtin-src"] textarea,
|
||||
body[data-theme-skin="builtin-src"] select {
|
||||
border-radius: 4px !important;
|
||||
transition: outline 150ms ease;
|
||||
}
|
||||
body[data-theme-skin="builtin-src"] input:not([type="checkbox"]):not([type="radio"]):not([type="range"]):focus-visible,
|
||||
body[data-theme-skin="builtin-src"] textarea:focus-visible,
|
||||
body[data-theme-skin="builtin-src"] select:focus-visible {
|
||||
outline: 2px solid var(--color-primary) !important;
|
||||
outline-offset: -2px !important;
|
||||
}
|
||||
|
||||
/* ── MD3 Cards: elevated (level 1), medium shape (12 dp) ─────── */
|
||||
body[data-theme-skin="builtin-src"] .bg-card {
|
||||
border-radius: 12px !important;
|
||||
box-shadow:
|
||||
0 1px 2px rgba(0, 0, 0, 0.10),
|
||||
0 1px 3px 1px rgba(0, 0, 0, 0.06) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] .bg-card {
|
||||
box-shadow:
|
||||
0 1px 2px rgba(0, 0, 0, 0.35),
|
||||
0 1px 3px 1px rgba(0, 0, 0, 0.20) !important;
|
||||
}
|
||||
|
||||
/* ── MD3 Menus / popovers / listboxes: extra-small (4 dp) ───── */
|
||||
body[data-theme-skin="builtin-src"] .bg-popover,
|
||||
body[data-theme-skin="builtin-src"] [role="listbox"] {
|
||||
border-radius: 4px !important;
|
||||
border: none !important;
|
||||
box-shadow:
|
||||
0 2px 6px 2px rgba(0, 0, 0, 0.15),
|
||||
0 1px 2px rgba(0, 0, 0, 0.30) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] .bg-popover,
|
||||
.dark body[data-theme-skin="builtin-src"] [role="listbox"] {
|
||||
box-shadow:
|
||||
0 2px 8px 2px rgba(0, 0, 0, 0.50),
|
||||
0 1px 2px rgba(0, 0, 0, 0.60) !important;
|
||||
}
|
||||
|
||||
/* ── MD3 Dialogs: extra-large shape (28 dp) ──────────────────── */
|
||||
body[data-theme-skin="builtin-src"] [role="dialog"] {
|
||||
border-radius: 28px !important;
|
||||
border: none !important;
|
||||
box-shadow:
|
||||
0 6px 10px 4px rgba(0, 0, 0, 0.15),
|
||||
0 2px 3px rgba(0, 0, 0, 0.30) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] [role="dialog"] {
|
||||
box-shadow:
|
||||
0 6px 10px 4px rgba(0, 0, 0, 0.50),
|
||||
0 2px 3px rgba(0, 0, 0, 0.60) !important;
|
||||
}
|
||||
|
||||
/* ── MD3 Menu items: 8 % on-surface state layer on hover ─────── */
|
||||
body[data-theme-skin="builtin-src"] [role="menu"] [role="menuitem"]:hover,
|
||||
body[data-theme-skin="builtin-src"] [role="menu"] [role="menuitem"]:focus,
|
||||
body[data-theme-skin="builtin-src"] [role="menu"] [role="menuitem"]:focus-visible {
|
||||
background-color: color-mix(in srgb, var(--color-foreground) 8%, transparent) !important;
|
||||
color: var(--color-foreground) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] [role="menu"] [role="menuitem"]:hover,
|
||||
.dark body[data-theme-skin="builtin-src"] [role="menu"] [role="menuitem"]:focus {
|
||||
background-color: color-mix(in srgb, var(--color-foreground) 10%, transparent) !important;
|
||||
}
|
||||
|
||||
/* ── MD3 Switch ──────────────────────────────────────────────── */
|
||||
/* M3 switch: unselected = outline + icon; selected = primary fill */
|
||||
body[data-theme-skin="builtin-src"] [role="switch"] {
|
||||
background-color: var(--color-input) !important;
|
||||
border: 2px solid var(--color-muted-foreground) !important;
|
||||
transition: background-color 150ms ease, border-color 150ms ease;
|
||||
}
|
||||
body[data-theme-skin="builtin-src"] [role="switch"] > span {
|
||||
background-color: var(--color-muted-foreground) !important;
|
||||
}
|
||||
body[data-theme-skin="builtin-src"] [role="switch"][aria-checked="true"] {
|
||||
background-color: var(--color-primary) !important;
|
||||
border-color: var(--color-primary) !important;
|
||||
}
|
||||
body[data-theme-skin="builtin-src"] [role="switch"][aria-checked="true"] > span {
|
||||
background-color: var(--color-primary-foreground) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] [role="switch"] {
|
||||
background-color: #3a2524 !important;
|
||||
border-color: var(--color-muted-foreground) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] [role="switch"][aria-checked="true"] {
|
||||
background-color: var(--color-primary) !important;
|
||||
border-color: var(--color-primary) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] [role="switch"][aria-checked="true"] > span {
|
||||
background-color: #1c1917 !important;
|
||||
}
|
||||
|
||||
/* ── Selected folder: M3 active-indicator treatment ─────────── */
|
||||
/* M3 uses a pill-shaped tonal container for the active nav item. */
|
||||
/* Bulwark's left-border accent becomes a 3 dp primary accent line */
|
||||
/* + secondary-container (--color-accent) fill + gentle corner. */
|
||||
body[data-theme-skin="builtin-src"] .bg-secondary.border-r .border-l-2.border-primary {
|
||||
border-left-width: 3px !important;
|
||||
border-left-color: var(--color-primary) !important;
|
||||
background-color: var(--color-accent) !important;
|
||||
border-radius: 0 12px 12px 0 !important;
|
||||
}
|
||||
|
||||
/* ── Login card: MD3 extra-large + SRC red top accent strip ───── */
|
||||
body[data-theme-skin="builtin-src"] .rounded-2xl.border.bg-background\\/80.backdrop-blur-sm {
|
||||
position: relative;
|
||||
background-color: var(--color-card) !important;
|
||||
border-color: rgba(213, 43, 30, 0.14) !important;
|
||||
border-radius: 28px !important;
|
||||
box-shadow:
|
||||
0 2px 8px rgba(0, 0, 0, 0.08),
|
||||
0 8px 32px rgba(0, 0, 0, 0.06) !important;
|
||||
overflow: hidden;
|
||||
}
|
||||
body[data-theme-skin="builtin-src"] .rounded-2xl.border.bg-background\\/80.backdrop-blur-sm::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
left: 0; right: 0; top: 0;
|
||||
height: 3px;
|
||||
background: linear-gradient(90deg, transparent, #d52b1e 30%, #d52b1e 70%, transparent);
|
||||
pointer-events: none;
|
||||
}
|
||||
/* Login page background: faint SRC red ambient wash */
|
||||
body[data-theme-skin="builtin-src"] .min-h-screen.bg-gradient-to-br {
|
||||
background-color: #f9f7f7 !important;
|
||||
background-image: radial-gradient(56rem 38rem at 50% -12%, rgba(213, 43, 30, 0.05), transparent 60%) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] .rounded-2xl.border.bg-background\\/80.backdrop-blur-sm {
|
||||
border-color: rgba(239, 68, 68, 0.18) !important;
|
||||
box-shadow:
|
||||
0 4px 16px rgba(0, 0, 0, 0.45),
|
||||
0 0 40px -20px rgba(239, 68, 68, 0.18) !important;
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] .rounded-2xl.border.bg-background\\/80.backdrop-blur-sm::before {
|
||||
background: linear-gradient(90deg, transparent, #ef4444 30%, #ef4444 70%, transparent);
|
||||
}
|
||||
.dark body[data-theme-skin="builtin-src"] .min-h-screen.bg-gradient-to-br {
|
||||
background-color: var(--color-background) !important;
|
||||
background-image: radial-gradient(56rem 38rem at 50% -12%, rgba(239, 68, 68, 0.07), transparent 60%) !important;
|
||||
}
|
||||
|
||||
/* ── Single-surface panes (M3 has no gradient empty states) ───── */
|
||||
body[data-theme-skin="builtin-src"] .bg-gradient-to-br.from-muted\\/30.to-muted\\/50 {
|
||||
background: var(--color-background) !important;
|
||||
}
|
||||
body[data-theme-skin="builtin-src"] .bg-muted\\/30 {
|
||||
background-color: var(--color-background) !important;
|
||||
}`;
|
||||
|
||||
export const BUILTIN_THEMES: InstalledTheme[] = [
|
||||
{
|
||||
id: 'builtin-vnclagoon',
|
||||
@@ -1114,13 +1338,15 @@ export const BUILTIN_THEMES: InstalledTheme[] = [
|
||||
{
|
||||
id: 'builtin-src',
|
||||
name: 'SRC',
|
||||
version: '1.0.0',
|
||||
version: '1.1.0',
|
||||
author: 'VNC',
|
||||
description: 'SRC Advisory brand theme — Swiss red on white, light-first',
|
||||
description: 'SRC Advisory brand theme — Swiss red on white, MD3 components, light-first',
|
||||
css: srcCSS,
|
||||
logoLightUrl: '/branding/src-logo.svg',
|
||||
logoDarkUrl: '/branding/src-logo.svg',
|
||||
skin: srcSkin,
|
||||
logoLightUrl: '/branding/SRC_Symbol.png',
|
||||
logoDarkUrl: '/branding/SRC_Symbol.png',
|
||||
variants: ['light', 'dark'],
|
||||
typography: { fontSans: '"DM Sans", system-ui, -apple-system, "Segoe UI", sans-serif' },
|
||||
enabled: true,
|
||||
builtIn: true,
|
||||
},
|
||||
|
||||
@@ -75,6 +75,7 @@ export class DemoJMAPClient implements IJMAPClient {
|
||||
getMaxDelayedSend(): number { return 30 * 24 * 60 * 60; }
|
||||
hasDelayedSend(): boolean { return true; }
|
||||
getEventSourceUrl(): string | null { return null; }
|
||||
getWebSocketUrl(): string | null { return null; }
|
||||
supportsEmailSubmission(): boolean { return true; }
|
||||
supportsQuota(): boolean { return true; }
|
||||
supportsVacationResponse(): boolean { return true; }
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Detects whether the app is running inside the VNCmail+ (Bulwark) Electron
|
||||
// desktop shell and wraps the native notification bridge that
|
||||
// electron/preload.ts exposes via contextBridge. Mirrors how lib/web-push.ts
|
||||
// mirrors the React Native push flow - same idea, different native API:
|
||||
// PushManager/service-worker there, Electron's own Notification API here.
|
||||
//
|
||||
// Web/PWA deployments never get `window.vnc` at all (contextBridge only
|
||||
// exists inside the Electron shell), so `isElectronShell()` is false there
|
||||
// and callers should keep using the lib/web-push.ts + public/sw.js path.
|
||||
// Wiring this bridge up to real mail-delivery events (JMAP WebSocket push
|
||||
// vs. polling) is a separate, later decision - this module is only the
|
||||
// plumbing.
|
||||
|
||||
export interface ShowNotificationOptions {
|
||||
body?: string;
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
export interface ShowNotificationResult {
|
||||
shown: boolean;
|
||||
}
|
||||
|
||||
export interface VncElectronBridge {
|
||||
isElectron: true;
|
||||
showNotification: (
|
||||
title: string,
|
||||
options?: ShowNotificationOptions,
|
||||
) => Promise<ShowNotificationResult>;
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
vnc?: VncElectronBridge;
|
||||
}
|
||||
}
|
||||
|
||||
export function isElectronShell(): boolean {
|
||||
return typeof window !== "undefined" && window.vnc?.isElectron === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shows a notification via Electron's native Notification API when running
|
||||
* inside the desktop shell. Resolves to false (never throws) when not
|
||||
* running in Electron, or when the main process reports notifications
|
||||
* unsupported on this OS/session - callers can fall back to the
|
||||
* service-worker push path (lib/web-push.ts) in that case.
|
||||
*/
|
||||
export async function showElectronNotification(
|
||||
title: string,
|
||||
options?: ShowNotificationOptions,
|
||||
): Promise<boolean> {
|
||||
if (!isElectronShell()) return false;
|
||||
const result = await window.vnc!.showNotification(title, options);
|
||||
return result.shown;
|
||||
}
|
||||
@@ -35,6 +35,7 @@ export interface IJMAPClient {
|
||||
getMaxDelayedSend(accountId?: string): number;
|
||||
hasDelayedSend(accountId?: string): boolean;
|
||||
getEventSourceUrl(): string | null;
|
||||
getWebSocketUrl(): string | null;
|
||||
supportsEmailSubmission(): boolean;
|
||||
supportsQuota(): boolean;
|
||||
supportsVacationResponse(): boolean;
|
||||
|
||||
+455
-8
@@ -3,6 +3,7 @@ import type { SieveScript, SieveCapabilities } from "./sieve-types";
|
||||
import type { IJMAPClient } from "./client-interface";
|
||||
import { toWildcardQuery } from "./search-utils";
|
||||
import { batched, itemsPerRequest } from "./request-limits";
|
||||
import { noteTransportFailure, noteTransportSuccess } from "./transport-health";
|
||||
import { debug } from "@/lib/debug";
|
||||
import { normalizeCalendarEventLike } from "@/lib/calendar-event-normalization";
|
||||
|
||||
@@ -683,11 +684,24 @@ export class JMAPClient implements IJMAPClient {
|
||||
try {
|
||||
response = await fetch(url, { ...init, headers });
|
||||
} catch (error) {
|
||||
// A `fetch` REJECTION - and only that - is a transport failure. Recorded so
|
||||
// the offline replica's read fallback can tell "the network is down" from
|
||||
// "the folder is empty", which the error-swallowing in getEmails/getEmail/
|
||||
// getMailboxes otherwise makes indistinguishable (see
|
||||
// lib/jmap/transport-health.ts). Deliberately NOT recorded for a 4xx/5xx or
|
||||
// a 429: in those cases the server answered, so it is reachable.
|
||||
noteTransportFailure();
|
||||
// Network error: retry once after brief delay (transient proxy/connection issues)
|
||||
if (this.reconnecting) throw error;
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
response = await fetch(url, { ...init, headers });
|
||||
try {
|
||||
response = await fetch(url, { ...init, headers });
|
||||
} catch (retryError) {
|
||||
noteTransportFailure();
|
||||
throw retryError;
|
||||
}
|
||||
}
|
||||
noteTransportSuccess();
|
||||
|
||||
// Handle 429 rate limiting - stop immediately, do not retry
|
||||
if (response.status === 429) {
|
||||
@@ -953,6 +967,36 @@ export class JMAPClient implements IJMAPClient {
|
||||
if (session.eventSourceUrl) {
|
||||
session.eventSourceUrl = this.rewriteSessionUrl(session.eventSourceUrl);
|
||||
}
|
||||
const wsCapability = session.capabilities?.["urn:ietf:params:jmap:websocket"] as
|
||||
| { url?: string }
|
||||
| undefined;
|
||||
if (wsCapability?.url) {
|
||||
wsCapability.url = this.rewriteWebSocketUrl(wsCapability.url);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Same reasoning as rewriteSessionUrl (a reverse proxy may advertise its
|
||||
* own internal hostname), but scheme-aware: unlike apiUrl/eventSourceUrl,
|
||||
* this URL is never touched by fetch() - it goes straight into `new
|
||||
* WebSocket(...)`, and a ws/wss URL can never share an origin string with
|
||||
* an http/https serverUrl even when the host is identical, so reusing
|
||||
* rewriteSessionUrl's plain origin-equality check would rewrite EVERY
|
||||
* websocket URL onto an http(s) scheme and break the constructor outright.
|
||||
*/
|
||||
private rewriteWebSocketUrl(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const server = new URL(this.serverUrl);
|
||||
const expectedScheme = server.protocol === "https:" ? "wss:" : "ws:";
|
||||
if (parsed.host === server.host && parsed.protocol === expectedScheme) {
|
||||
return url;
|
||||
}
|
||||
const pathAndRest = url.slice(url.indexOf("/", url.indexOf("//") + 2));
|
||||
return `${expectedScheme}//${server.host}${pathAndRest}`;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
private async request(methodCalls: JMAPMethodCall[], using?: string[]): Promise<JMAPResponse> {
|
||||
@@ -3761,6 +3805,22 @@ export class JMAPClient implements IJMAPClient {
|
||||
return this.session.eventSourceUrl || coreCapability?.eventSourceUrl || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* RFC 8887 (JMAP over WebSocket) push endpoint, advertised under the
|
||||
* `urn:ietf:params:jmap:websocket` capability (not a root session field
|
||||
* like eventSourceUrl - it's nested the same way every other JMAP
|
||||
* extension capability is). Rewritten to the client's own server host in
|
||||
* rewriteSessionUrls() at connect time, same reasoning as apiUrl/
|
||||
* downloadUrl/eventSourceUrl. Returns null for servers that don't
|
||||
* advertise it - callers fall back to SSE/polling.
|
||||
*/
|
||||
getWebSocketUrl(): string | null {
|
||||
const wsCapability = this.capabilities["urn:ietf:params:jmap:websocket"] as
|
||||
| { url?: string; supportsPush?: boolean }
|
||||
| undefined;
|
||||
return wsCapability?.url || null;
|
||||
}
|
||||
|
||||
getAccountId(): string {
|
||||
return this.accountId;
|
||||
}
|
||||
@@ -5982,12 +6042,51 @@ export class JMAPClient implements IJMAPClient {
|
||||
private visibilityHandler: (() => void) | null = null;
|
||||
private onlineHandler: (() => void) | null = null;
|
||||
|
||||
// JMAP-over-WebSocket (RFC 8887) push - preferred over SSE when the server
|
||||
// advertises it (getWebSocketUrl()), since it's the transport the desktop
|
||||
// shell's main process eventually wants for background/no-window
|
||||
// notifications (see electron/preload.ts's showNotification bridge).
|
||||
// Falls back to the existing SSE/polling chain below when unsupported OR
|
||||
// when the handshake itself keeps failing (see wsPermanentlyDisabled).
|
||||
//
|
||||
// KNOWN LIMITATION, confirmed empirically against the sandbox server this
|
||||
// was built against (stalwart.sandbox.vnc.de): its /jmap/ws endpoint
|
||||
// requires the same HTTP Basic/Bearer Authorization header as every other
|
||||
// JMAP endpoint on the WebSocket UPGRADE request itself (curling it with
|
||||
// no Authorization header returns a plain 401 before any WS frame is
|
||||
// possible). The browser WebSocket constructor has no way to attach
|
||||
// custom headers to that handshake (a WHATWG spec restriction, not an
|
||||
// Electron/browser quirk - credentials in the URL are actively rejected
|
||||
// too), so from this renderer-side client there is no way to satisfy that
|
||||
// auth requirement. Against a server with this exact auth model, every
|
||||
// connection attempt below will fail at the handshake and the circuit
|
||||
// breaker (wsPermanentlyDisabled) will fall back to SSE after a few quick
|
||||
// retries - which is not a bug in this code, it is what actually happens
|
||||
// on the wire. It's still implemented for real (not stubbed) because (a)
|
||||
// it's fully spec-correct and will light up automatically against any
|
||||
// server whose WS endpoint doesn't have this requirement - e.g. one
|
||||
// sitting behind a proxy that authenticates via cookies instead - with no
|
||||
// further changes, and (b) the alternative (opening it from Electron's
|
||||
// main process via a header-capable client like the `ws` package) would
|
||||
// mean piping raw credentials from the renderer to the main process over
|
||||
// IPC, which is a materially bigger security-sensitive change than what
|
||||
// was scoped here.
|
||||
private ws: WebSocket | null = null;
|
||||
private wsReconnectTimeout: NodeJS.Timeout | null = null;
|
||||
private wsReconnectAttempts: number = 0;
|
||||
private wsConsecutiveFailures: number = 0;
|
||||
private wsPermanentlyDisabled: boolean = false;
|
||||
private wsHeartbeatTimer: NodeJS.Timeout | null = null;
|
||||
private lastWSActivity: number = 0;
|
||||
|
||||
private static readonly STATE_TYPE_MAP: Record<string, string> = {
|
||||
'Mailbox/get': 'Mailbox',
|
||||
'Email/get': 'Email',
|
||||
'Calendar/get': 'Calendar',
|
||||
'CalendarEvent/get': 'CalendarEvent',
|
||||
'SieveScript/get': 'SieveScript',
|
||||
'ContactCard/get': 'ContactCard',
|
||||
'FileNode/get': 'FileNode',
|
||||
};
|
||||
|
||||
private static readonly POLLING_INTERVAL = 3_000;
|
||||
@@ -5998,20 +6097,315 @@ export class JMAPClient implements IJMAPClient {
|
||||
private static readonly SSE_RECONNECT_DELAY = 3_000;
|
||||
private static readonly SSE_PING_TIMEOUT = 90_000; // 3x the 30s ping interval
|
||||
|
||||
// Exponential backoff with full jitter (0..cap), doubling from a 200ms
|
||||
// base and capping at 5s.
|
||||
//
|
||||
// Deliberately much tighter than a "normal" reconnect ladder (something
|
||||
// like 1s/30s would be the textbook default for a flaky network) - and
|
||||
// tuned from a real, measured failure mode, not guessed: the auth
|
||||
// limitation described above fails FAST and DETERMINISTICALLY (the
|
||||
// handshake is rejected before the socket ever opens, in well under a
|
||||
// second, every single time), not slowly. Verified empirically (see
|
||||
// integration/tests/11-electron-notification.spec.ts's development) that
|
||||
// the original 1s-base/30s-cap/5-attempt ladder let the circuit breaker
|
||||
// take up to ~31s to trip, during which there is NO live push at all
|
||||
// (WS hasn't succeeded and hasn't given up yet, so SSE never even starts
|
||||
// connecting) - a real mail delivery landing in that window was missed
|
||||
// entirely, since SSE only streams future changes and does no catch-up
|
||||
// fetch on connect. This tighter ladder closes that gap to a fraction of
|
||||
// a second for the fast-fail case while remaining exactly as protective
|
||||
// for a genuinely slow/flaky network: a hanging attempt is still bounded
|
||||
// by the browser's own WebSocket connect timeout regardless of these
|
||||
// constants, which govern only the GAP between attempts, not how long a
|
||||
// single attempt is allowed to hang.
|
||||
private static readonly WS_RECONNECT_BASE_DELAY = 200;
|
||||
private static readonly WS_RECONNECT_MAX_DELAY = 5_000;
|
||||
// App-level heartbeat: a WebSocket can sit in "open" readyState for a long
|
||||
// time after the underlying network path is actually gone (sleep, network
|
||||
// switch, a NAT/proxy that silently drops idle connections) - TCP alone
|
||||
// won't always surface that promptly. Send a lightweight JMAP request
|
||||
// every 30s and force-reconnect if nothing (heartbeat response OR a real
|
||||
// push) has arrived within 3x that window, mirroring the SSE ping monitor
|
||||
// above.
|
||||
private static readonly WS_HEARTBEAT_INTERVAL = 30_000;
|
||||
private static readonly WS_ACTIVITY_TIMEOUT = 90_000;
|
||||
// Give up on WS for this client instance after this many CONSECUTIVE
|
||||
// attempts that never reach "open" (a connection that opened fine and
|
||||
// later dropped does not count - see connectWebSocket's openedSuccessfully
|
||||
// tracking). Bounds the cost of the auth limitation described above to a
|
||||
// handful of quick handshake attempts (with the tightened backoff above,
|
||||
// well under a second in the common fast-fail case) instead of retrying a
|
||||
// request that can never succeed, forever, for the lifetime of the session.
|
||||
private static readonly WS_MAX_CONSECUTIVE_FAILURES = 3;
|
||||
|
||||
/** getWebSocketUrl(), gated by the circuit breaker above. */
|
||||
private effectiveWebSocketUrl(): string | null {
|
||||
return this.wsPermanentlyDisabled ? null : this.getWebSocketUrl();
|
||||
}
|
||||
|
||||
setupPushNotifications(): boolean {
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
// SSE covers the primary account only; keep shared accounts fresh too.
|
||||
const wsUrl = this.effectiveWebSocketUrl();
|
||||
if (wsUrl) {
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.connectWebSocket(wsUrl);
|
||||
// Prime the polling baseline (pollingStates) in parallel with the WS
|
||||
// attempt, not just for shared/secondary accounts below - if WS ends
|
||||
// up failing and falling back (fallbackFromWebSocket()), this is what
|
||||
// lets that fallback reconcile anything that changed to the PRIMARY
|
||||
// account while WS was still churning through retries. Without an
|
||||
// early baseline, a change in that window would be silently missed
|
||||
// entirely: SSE only streams changes from the moment it connects
|
||||
// onward (no catch-up on connect), so the one thing that CAN catch up
|
||||
// is a diff against a state snapshot taken before the gap started.
|
||||
void this.fetchCurrentStates();
|
||||
// Not confirmed either way whether this server's WebSocket push fans
|
||||
// out to shared/secondary accounts or, like Stalwart's SSE, covers the
|
||||
// primary account only - keep the same secondary poll running under
|
||||
// WS that SSE already needed, rather than assume broader coverage and
|
||||
// risk shared-account counters going stale.
|
||||
this.startSecondaryAccountPoll();
|
||||
} else {
|
||||
// The fallback poll already covers every session account.
|
||||
this.startPollingFallback();
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
// SSE covers the primary account only; keep shared accounts fresh too.
|
||||
this.startSecondaryAccountPoll();
|
||||
} else {
|
||||
// The fallback poll already covers every session account.
|
||||
this.startPollingFallback();
|
||||
}
|
||||
}
|
||||
this.setupBrowserEventListeners();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the RFC 8887 JMAP-over-WebSocket connection and subscribes to
|
||||
* push for every data type (`WebSocketPushEnable` with dataTypes: null).
|
||||
* Reconnect on close/error is handled by scheduleWSReconnect() below with
|
||||
* exponential backoff - this method only ever represents a single
|
||||
* connection attempt.
|
||||
*/
|
||||
private connectWebSocket(wsUrl: string): void {
|
||||
if (this.isRateLimited()) {
|
||||
this.scheduleWSReconnect();
|
||||
return;
|
||||
}
|
||||
|
||||
let socket: WebSocket;
|
||||
try {
|
||||
socket = new WebSocket(wsUrl, "jmap");
|
||||
} catch {
|
||||
// New URL()-level failures (malformed URL) - retry later in case a
|
||||
// session refresh fixes it; getWebSocketUrl() re-reads capabilities
|
||||
// fresh on every attempt.
|
||||
this.scheduleWSReconnect();
|
||||
return;
|
||||
}
|
||||
|
||||
this.ws = socket;
|
||||
const isCurrent = () => this.ws === socket;
|
||||
// Tracks whether THIS specific attempt ever reached "open" - a socket
|
||||
// that opened fine and dropped later (real network blip on an
|
||||
// established connection) must not count toward the circuit breaker the
|
||||
// same way a handshake that never completes does (see
|
||||
// wsPermanentlyDisabled's declaration above for why the latter needs
|
||||
// one at all).
|
||||
let openedSuccessfully = false;
|
||||
|
||||
socket.addEventListener("open", () => {
|
||||
if (!isCurrent()) return;
|
||||
openedSuccessfully = true;
|
||||
// A real connection succeeded - both counters reset: the backoff
|
||||
// ladder no longer applies to whatever eventually causes the NEXT
|
||||
// disconnect, and the "give up on WS entirely" counter only tracks
|
||||
// CONSECUTIVE handshake failures.
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.wsConsecutiveFailures = 0;
|
||||
this.lastWSActivity = Date.now();
|
||||
this.startWSHeartbeat(socket);
|
||||
try {
|
||||
socket.send(JSON.stringify({ "@type": "WebSocketPushEnable", dataTypes: null }));
|
||||
} catch {
|
||||
// send() can throw if the socket already closed between "open"
|
||||
// firing and this line running - the "close" handler below will
|
||||
// schedule a reconnect regardless.
|
||||
}
|
||||
});
|
||||
|
||||
socket.addEventListener("message", (event) => {
|
||||
if (!isCurrent()) return;
|
||||
this.lastWSActivity = Date.now();
|
||||
this.processWebSocketMessage(typeof event.data === "string" ? event.data : "");
|
||||
});
|
||||
|
||||
socket.addEventListener("close", () => {
|
||||
if (!isCurrent()) return;
|
||||
this.stopWSHeartbeat();
|
||||
this.ws = null;
|
||||
if (this.intentionallyDisconnected) return;
|
||||
|
||||
if (!openedSuccessfully) {
|
||||
this.wsConsecutiveFailures += 1;
|
||||
if (this.wsConsecutiveFailures >= JMAPClient.WS_MAX_CONSECUTIVE_FAILURES) {
|
||||
// The handshake itself is what's failing, repeatedly - most
|
||||
// commonly (confirmed against this client's own reference
|
||||
// server) because the WS endpoint requires an Authorization
|
||||
// header the browser WebSocket API cannot attach. Retrying that
|
||||
// forever would just hammer the server every ~30s with a request
|
||||
// that can never succeed from here. Give up on WS for the rest of
|
||||
// this client instance's life and stay on SSE/polling, which
|
||||
// don't have this limitation.
|
||||
this.wsPermanentlyDisabled = true;
|
||||
console.warn(
|
||||
'[JMAP] WebSocket push failed to establish after repeated attempts; falling back to SSE/polling for this session.',
|
||||
);
|
||||
this.fallbackFromWebSocket();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
this.scheduleWSReconnect();
|
||||
});
|
||||
|
||||
// WebSocket always fires "close" right after "error" - the reconnect
|
||||
// logic lives entirely in the "close" handler above so there is exactly
|
||||
// one path that schedules a retry, not two racing each other.
|
||||
}
|
||||
|
||||
/** Whatever push transport SSE would have used, now that WS has given up. */
|
||||
private fallbackFromWebSocket(): void {
|
||||
void this.reconcileAfterWebSocketFallback();
|
||||
}
|
||||
|
||||
/**
|
||||
* Diffs against the baseline setupPushNotifications() primed via
|
||||
* fetchCurrentStates() when the WS attempt began - BEFORE either branch
|
||||
* below gets a chance to erase that opportunity (startPollingFallback()
|
||||
* unconditionally overwrites the same baseline via its own
|
||||
* fetchCurrentStates() call; connectSSE() only ever streams changes from
|
||||
* the moment it connects onward, no catch-up). This is what catches a
|
||||
* real mail delivery (or any other tracked change) that happened to the
|
||||
* primary account while WS was still churning through retries, which
|
||||
* neither of those two paths would otherwise ever notice - confirmed as a
|
||||
* real, not theoretical, gap during this feature's own development (see
|
||||
* the WS_RECONNECT_BASE_DELAY comment above).
|
||||
*
|
||||
* Not airtight: if the early fetchCurrentStates() from
|
||||
* setupPushNotifications() hasn't itself completed yet by the time this
|
||||
* runs, there's nothing to diff against and this call just establishes
|
||||
* the baseline instead of detecting drift. In practice that race needs a
|
||||
* pathologically slow state-fetch racing an unusually fast WS failure,
|
||||
* and the tightened backoff above (worst case ~1.75s to exhaust 3
|
||||
* attempts) gives that fetch a lot more room to finish first than the
|
||||
* original 31s-worst-case ladder did.
|
||||
*/
|
||||
private async reconcileAfterWebSocketFallback(): Promise<void> {
|
||||
await this.checkForStateChanges();
|
||||
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
this.startSecondaryAccountPoll();
|
||||
} else {
|
||||
this.startPollingFallback();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses one WebSocket text frame. Per RFC 8887 the server can send
|
||||
* Response, StateChange, or PushState frames; only StateChange is
|
||||
* consumed today (method calls aren't yet routed over this socket -
|
||||
* request()/authenticatedFetch() still uses plain HTTP), so anything else
|
||||
* is silently ignored rather than treated as an error.
|
||||
*/
|
||||
private processWebSocketMessage(raw: string): void {
|
||||
if (!raw) return;
|
||||
let message: { "@type"?: string; changed?: StateChange["changed"] } | null = null;
|
||||
try {
|
||||
message = JSON.parse(raw);
|
||||
} catch {
|
||||
return; // malformed frame - ignore, matches processSSEEvent's handling
|
||||
}
|
||||
if (message?.["@type"] === "StateChange" && message.changed) {
|
||||
this.stateChangeCallback?.({ "@type": "StateChange", changed: message.changed });
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleWSReconnect(): void {
|
||||
if (this.intentionallyDisconnected) return;
|
||||
if (this.wsReconnectTimeout) return; // already scheduled - don't stack retries
|
||||
|
||||
const wsUrl = this.effectiveWebSocketUrl();
|
||||
if (!wsUrl) {
|
||||
// Either the server capability disappeared (e.g. a session refresh
|
||||
// dropped WebSocket support) or the circuit breaker already tripped -
|
||||
// fall back to whatever push transport is still available instead of
|
||||
// retrying a URL that's gone or a handshake that won't succeed.
|
||||
this.fallbackFromWebSocket();
|
||||
return;
|
||||
}
|
||||
|
||||
const attempt = this.wsReconnectAttempts;
|
||||
this.wsReconnectAttempts += 1;
|
||||
const exponential = JMAPClient.WS_RECONNECT_BASE_DELAY * Math.pow(2, attempt);
|
||||
const cap = Math.min(exponential, JMAPClient.WS_RECONNECT_MAX_DELAY);
|
||||
// Full jitter (uniform 0..cap) rather than a fixed exponential delay -
|
||||
// spreads reconnect attempts out after a shared network blip (proxy
|
||||
// restart, wifi handoff affecting every open tab/window at once)
|
||||
// instead of having them all retry in lockstep.
|
||||
const delay = Math.random() * cap;
|
||||
|
||||
this.wsReconnectTimeout = setTimeout(() => {
|
||||
this.wsReconnectTimeout = null;
|
||||
if (this.isRateLimited()) {
|
||||
this.scheduleWSReconnect();
|
||||
return;
|
||||
}
|
||||
this.connectWebSocket(wsUrl);
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private startWSHeartbeat(socket: WebSocket): void {
|
||||
this.stopWSHeartbeat();
|
||||
this.wsHeartbeatTimer = setInterval(() => {
|
||||
if (this.ws !== socket) return;
|
||||
if (Date.now() - this.lastWSActivity > JMAPClient.WS_ACTIVITY_TIMEOUT) {
|
||||
// Silently dead connection (sleep/network switch/idle proxy) - the
|
||||
// socket can still report readyState OPEN long after the underlying
|
||||
// path is gone. Force-close; the "close" handler schedules the
|
||||
// reconnect via the normal backoff path.
|
||||
this.stopWSHeartbeat();
|
||||
try {
|
||||
socket.close();
|
||||
} catch {
|
||||
// Already closing/closed - the "close" handler (if it hasn't
|
||||
// already run) will still fire and take care of reconnecting.
|
||||
}
|
||||
return;
|
||||
}
|
||||
try {
|
||||
socket.send(JSON.stringify({
|
||||
"@type": "Request",
|
||||
requestId: `ws-heartbeat-${Date.now()}`,
|
||||
using: ["urn:ietf:params:jmap:core"],
|
||||
methodCalls: [["Core/echo", {}, "0"]],
|
||||
}));
|
||||
} catch {
|
||||
// send() failing means the socket is already dead - the activity
|
||||
// timeout above will catch it on the next tick if "close" doesn't
|
||||
// fire first.
|
||||
}
|
||||
}, JMAPClient.WS_HEARTBEAT_INTERVAL);
|
||||
}
|
||||
|
||||
private stopWSHeartbeat(): void {
|
||||
if (this.wsHeartbeatTimer) {
|
||||
clearInterval(this.wsHeartbeatTimer);
|
||||
this.wsHeartbeatTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Slow poll of the session's shared/secondary accounts, run in parallel with
|
||||
* SSE (which never reports them). Skipped when there are no shared accounts,
|
||||
@@ -6210,6 +6604,23 @@ export class JMAPClient implements IJMAPClient {
|
||||
);
|
||||
}
|
||||
|
||||
// Contacts and files get no push at all today (mail-index's event-driven
|
||||
// reindex depends on this poll to notice them when SSE/WS isn't
|
||||
// available) - mirrors the Calendar branch above, same accountId caveat.
|
||||
if (this.supportsContacts()) {
|
||||
using.push('urn:ietf:params:jmap:contacts');
|
||||
methodCalls.push(
|
||||
['ContactCard/get', { accountId: this.getContactsAccountId(), ids: [], properties: ['id'] }, 'f'],
|
||||
);
|
||||
}
|
||||
|
||||
if (this.hasCapability('urn:ietf:params:jmap:filenode')) {
|
||||
using.push('urn:ietf:params:jmap:filenode');
|
||||
methodCalls.push(
|
||||
['FileNode/get', { accountId: this.getFilesAccountId(), ids: [], properties: ['id'] }, 'g'],
|
||||
);
|
||||
}
|
||||
|
||||
return { using, methodCalls };
|
||||
}
|
||||
|
||||
@@ -6310,6 +6721,27 @@ export class JMAPClient implements IJMAPClient {
|
||||
this.eventSource = null;
|
||||
}
|
||||
this.stopSSEPingMonitor();
|
||||
if (this.wsReconnectTimeout) {
|
||||
clearTimeout(this.wsReconnectTimeout);
|
||||
this.wsReconnectTimeout = null;
|
||||
}
|
||||
this.stopWSHeartbeat();
|
||||
if (this.ws) {
|
||||
// Null out this.ws BEFORE close() so the "close" event handler's
|
||||
// isCurrent() check (this.ws === socket) sees a mismatch once the
|
||||
// event fires and skips scheduling a reconnect - this is an
|
||||
// intentional teardown, not a dropped connection.
|
||||
const socket = this.ws;
|
||||
this.ws = null;
|
||||
try {
|
||||
socket.close();
|
||||
} catch {
|
||||
// Already closing/closed.
|
||||
}
|
||||
}
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.wsConsecutiveFailures = 0;
|
||||
this.wsPermanentlyDisabled = false;
|
||||
this.cleanupBrowserEventListeners();
|
||||
this.stateChangeCallback = null;
|
||||
this.pollingStates = {};
|
||||
@@ -6350,7 +6782,22 @@ export class JMAPClient implements IJMAPClient {
|
||||
|
||||
if (typeof window !== 'undefined') {
|
||||
this.onlineHandler = () => {
|
||||
// Network reconnected - reconnect SSE or force a poll
|
||||
// Network reconnected - reconnect WS/SSE or force a poll. Don't
|
||||
// make the user wait through whatever backoff delay was already in
|
||||
// flight from repeated failures while offline - the network is
|
||||
// confirmed back, so retry immediately.
|
||||
const wsUrl = this.effectiveWebSocketUrl();
|
||||
if (wsUrl) {
|
||||
if (!this.ws) {
|
||||
if (this.wsReconnectTimeout) {
|
||||
clearTimeout(this.wsReconnectTimeout);
|
||||
this.wsReconnectTimeout = null;
|
||||
}
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.connectWebSocket(wsUrl);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl && !this.sseAbortController) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// A single monotonic counter of JMAP TRANSPORT failures.
|
||||
//
|
||||
// WHY THIS EXISTS. The offline replica is a read-path FALLBACK, and to be one it
|
||||
// has to know that a read genuinely failed. `lib/jmap/client.ts` makes that
|
||||
// impossible to see from the outside: its read methods swallow their own errors
|
||||
// and return plausible-looking success. `getEmails()` returns
|
||||
// `{ emails: [], hasMore: false, total: 0 }`, so a dead network is
|
||||
// indistinguishable from an empty folder. `getEmail()` returns `null`.
|
||||
// `getMailboxes()` returns a SYNTHETIC single Inbox. Falling back on those shapes
|
||||
// alone would mean serving stale replica rows for a folder the user had genuinely
|
||||
// just emptied.
|
||||
//
|
||||
// So `authenticatedFetch` bumps this counter when, and only when, `fetch` itself
|
||||
// rejects - not on a 4xx, not on a 429 (that is a rate limit, and the server is
|
||||
// plainly reachable), not on a JMAP method error. The fallback layer samples the
|
||||
// counter before and after a call: a suspicious result PLUS an increment during
|
||||
// that exact call is a transport failure. Either signal alone is not enough.
|
||||
//
|
||||
// Module-level rather than per-client on purpose: it answers "is the network
|
||||
// working right now", which is a property of the machine, not of one account's
|
||||
// client instance.
|
||||
|
||||
let failures = 0;
|
||||
let lastFailureAt = 0;
|
||||
let lastSuccessAt = 0;
|
||||
|
||||
/** Called only when `fetch` itself rejects. Never for an HTTP status. */
|
||||
export function noteTransportFailure(): void {
|
||||
failures++;
|
||||
lastFailureAt = Date.now();
|
||||
}
|
||||
|
||||
export function noteTransportSuccess(): void {
|
||||
lastSuccessAt = Date.now();
|
||||
}
|
||||
|
||||
/** Monotonic. Sample before and after a call to attribute a failure to it. */
|
||||
export function transportFailureCount(): number {
|
||||
return failures;
|
||||
}
|
||||
|
||||
export function transportHealth(): {
|
||||
failures: number;
|
||||
lastFailureAt: number;
|
||||
lastSuccessAt: number;
|
||||
/** Best-effort "probably offline": a failure more recent than any success. */
|
||||
likelyOffline: boolean;
|
||||
} {
|
||||
return {
|
||||
failures,
|
||||
lastFailureAt,
|
||||
lastSuccessAt,
|
||||
likelyOffline: lastFailureAt > lastSuccessAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Test-only reset. */
|
||||
export function resetTransportHealth(): void {
|
||||
failures = 0;
|
||||
lastFailureAt = 0;
|
||||
lastSuccessAt = 0;
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
// Renderer-side client for the encrypted local search index.
|
||||
//
|
||||
// The index is EVENT-DRIVEN: the renderer already holds the live JMAP push
|
||||
// connection (WebSocket -> SSE -> polling, `lib/jmap/client.ts`'s
|
||||
// setupPushNotifications), so the moment a StateChange announces new mail, a
|
||||
// calendar change, a contact edit or a file upload, this posts to the reindex
|
||||
// route. No polling loop, no background worker, no long-lived credential -
|
||||
// just one more authenticated fetch from the place the push already arrives.
|
||||
//
|
||||
// Every function here is best-effort and never throws: a search index failing
|
||||
// to update must never break the mail UI.
|
||||
|
||||
import { apiFetch } from '@/lib/browser-navigation';
|
||||
import { debug } from '@/lib/debug';
|
||||
import type { StateChange } from '@/lib/jmap/types';
|
||||
|
||||
export type IndexContentType = 'mail' | 'calendar' | 'contact' | 'file';
|
||||
|
||||
export interface IndexRunResult {
|
||||
ok: boolean;
|
||||
written?: Partial<Record<IndexContentType, number>>;
|
||||
skipped?: IndexContentType[];
|
||||
errors?: Array<{ contentType: IndexContentType; message: string }>;
|
||||
durationMs?: number;
|
||||
/** Set when the feature isn't available (not the desktop shell, no keyring, no binding). */
|
||||
unavailable?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps JMAP `StateChange` type keys onto our content types.
|
||||
*
|
||||
* The transport is already type-generic - the WebSocket handler
|
||||
* (`client.ts:6308-6316`) and the SSE handler (`:6505`) pass the whole
|
||||
* `changed` map through untouched, and the WS subscribes with
|
||||
* `dataTypes: null` (every type) - so anything the server pushes arrives here.
|
||||
*
|
||||
* `Mailbox` is deliberately NOT mapped: a Mailbox state change is usually just
|
||||
* an unread-count move, and it fires constantly. `Email` covers the cases that
|
||||
* change indexable content.
|
||||
*/
|
||||
const STATE_TYPE_TO_CONTENT: Record<string, IndexContentType> = {
|
||||
Email: 'mail',
|
||||
Calendar: 'calendar',
|
||||
CalendarEvent: 'calendar',
|
||||
ContactCard: 'contact',
|
||||
AddressBook: 'contact',
|
||||
FileNode: 'file',
|
||||
};
|
||||
|
||||
export function contentTypesFromStateChange(change: StateChange): IndexContentType[] {
|
||||
const out = new Set<IndexContentType>();
|
||||
for (const perAccount of Object.values(change.changed ?? {})) {
|
||||
for (const stateType of Object.keys(perAccount ?? {})) {
|
||||
const mapped = STATE_TYPE_TO_CONTENT[stateType];
|
||||
if (mapped) out.add(mapped);
|
||||
}
|
||||
}
|
||||
return [...out];
|
||||
}
|
||||
|
||||
export interface IndexRequestOptions {
|
||||
types?: readonly IndexContentType[];
|
||||
/**
|
||||
* Per-type ids to index. Supply them whenever the renderer already knows
|
||||
* which objects changed - it turns the call into a couple of `Foo/get`s
|
||||
* instead of a windowed query. Mail is the frequent case and the one where
|
||||
* this matters.
|
||||
*/
|
||||
ids?: Partial<Record<IndexContentType, string[]>>;
|
||||
/** Backfill the recent window for every supported type, and prune. */
|
||||
catchUp?: boolean;
|
||||
/** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */
|
||||
slot?: number;
|
||||
}
|
||||
|
||||
let inFlight: Promise<IndexRunResult> | null = null;
|
||||
/** Set once the server says the feature isn't there, so we stop asking. */
|
||||
let knownUnavailable = false;
|
||||
|
||||
/**
|
||||
* Posts one index request. Single-flighted: a burst of deliveries coalesces
|
||||
* into the in-flight call rather than queueing N overlapping SQLite writers.
|
||||
*/
|
||||
export async function requestIndex(options: IndexRequestOptions = {}): Promise<IndexRunResult> {
|
||||
if (knownUnavailable) return { ok: false, unavailable: true };
|
||||
if (inFlight) return inFlight;
|
||||
|
||||
const query = typeof options.slot === 'number' ? `?slot=${options.slot}` : '';
|
||||
const run = (async (): Promise<IndexRunResult> => {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/reindex${query}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
types: options.types,
|
||||
ids: options.ids,
|
||||
catchUp: options.catchUp === true,
|
||||
}),
|
||||
});
|
||||
|
||||
// 404 = not the desktop shell (or the feature is gated off). Permanent for
|
||||
// this page load; stop asking so a busy mailbox doesn't post per delivery.
|
||||
if (response.status === 404) {
|
||||
knownUnavailable = true;
|
||||
return { ok: false, unavailable: true };
|
||||
}
|
||||
if (response.status === 503) {
|
||||
// No keyring / no native binding / no key channel. Also permanent for
|
||||
// this session, and the message is worth surfacing in Settings.
|
||||
knownUnavailable = true;
|
||||
const body = await response.json().catch(() => ({}));
|
||||
return { ok: false, unavailable: true, error: body?.error };
|
||||
}
|
||||
if (!response.ok) {
|
||||
const body = await response.json().catch(() => ({}));
|
||||
return { ok: false, error: body?.error || `HTTP ${response.status}` };
|
||||
}
|
||||
const body = await response.json();
|
||||
debug.log('push', '[index] reindex done', body?.written, body?.errors);
|
||||
return {
|
||||
ok: true,
|
||||
written: body?.written,
|
||||
skipped: body?.skipped,
|
||||
errors: body?.errors,
|
||||
durationMs: body?.durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
||||
} finally {
|
||||
inFlight = null;
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
/**
|
||||
* The event-driven entry point, called from the push handler.
|
||||
*
|
||||
* `mailIds` lets the caller hand over the ids it already has (the refreshed
|
||||
* mailbox page), so the frequent mail case costs one `Email/get` rather than a
|
||||
* 30-day query. The other three types are rare events (a contact edit, a file
|
||||
* upload, a calendar change), so they fall back to their own bounded queries.
|
||||
*/
|
||||
export function indexOnStateChange(
|
||||
change: StateChange,
|
||||
opts: { mailIds?: string[]; slot?: number } = {},
|
||||
): void {
|
||||
if (knownUnavailable) return;
|
||||
const types = contentTypesFromStateChange(change);
|
||||
if (types.length === 0) return;
|
||||
|
||||
const ids: Partial<Record<IndexContentType, string[]>> = {};
|
||||
if (types.includes('mail') && opts.mailIds && opts.mailIds.length > 0) {
|
||||
ids.mail = opts.mailIds.slice(0, 100);
|
||||
}
|
||||
|
||||
// Fire-and-forget on purpose: this runs inside the push handler, and the mail
|
||||
// UI must not wait on a search index.
|
||||
void requestIndex({ types, ids: Object.keys(ids).length > 0 ? ids : undefined, slot: opts.slot });
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch-time catch-up: backfills whatever changed while the app was closed,
|
||||
* for which no push event was ever delivered. Also the recovery path for the
|
||||
* polling transport, which has no signal for contacts or files at all
|
||||
* (`client.ts`'s buildStatePollingRequest polls Mailbox/Email/Calendar/
|
||||
* CalendarEvent/SieveScript only).
|
||||
*/
|
||||
export async function catchUpIndex(slot?: number): Promise<IndexRunResult> {
|
||||
return requestIndex({ catchUp: true, slot });
|
||||
}
|
||||
|
||||
export interface IndexStats {
|
||||
contentType: string;
|
||||
count: number;
|
||||
newest: string | null;
|
||||
indexedAt: number | null;
|
||||
}
|
||||
|
||||
/** Reads per-type counts without searching. Used by the Settings panel. */
|
||||
export async function fetchIndexStats(slot?: number): Promise<IndexStats[] | null> {
|
||||
const slotQuery = typeof slot === 'number' ? `&slot=${slot}` : '';
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/search?stats=true${slotQuery}`);
|
||||
if (!response.ok) return null;
|
||||
const body = await response.json();
|
||||
return Array.isArray(body?.stats) ? (body.stats as IndexStats[]) : [];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resets the "don't ask again" latch - e.g. after the user signs in again. */
|
||||
export function resetIndexAvailability(): void {
|
||||
knownUnavailable = false;
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type {
|
||||
CalendarEvent, CalendarParticipant, ContactCard, Email, EmailBodyPart, FileNode,
|
||||
} from '@/lib/jmap/types';
|
||||
import {
|
||||
contactDisplayName, emailBodyText, extractCalendarEvent, extractContact, extractFile,
|
||||
extractMail, htmlToText, MAX_BODY_CHARS, normaliseText,
|
||||
} from '../extract';
|
||||
import { buildFilePaths } from '../jmap';
|
||||
|
||||
describe('htmlToText', () => {
|
||||
it('drops script and style CONTENT, not just the tags', () => {
|
||||
// The important case: a naive `<[^>]+>` strip leaves the script body behind
|
||||
// as searchable text, so a page full of JS would pollute the index.
|
||||
const out = htmlToText('<p>Hello</p><script>var secretToken = "abc123";</script><style>.a{color:red}</style>');
|
||||
expect(out).toContain('Hello');
|
||||
expect(out).not.toContain('secretToken');
|
||||
expect(out).not.toContain('abc123');
|
||||
expect(out).not.toContain('color:red');
|
||||
});
|
||||
|
||||
it('turns block boundaries into newlines and decodes entities', () => {
|
||||
expect(htmlToText('<p>one</p><p>two</p>')).toBe('one\ntwo');
|
||||
expect(htmlToText('a<br>b')).toBe('a\nb');
|
||||
expect(htmlToText('R&D <tag> "q" x')).toBe('R&D <tag> "q" x');
|
||||
expect(htmlToText('€10 €20')).toBe('€10 €20');
|
||||
});
|
||||
|
||||
it('ignores comments and out-of-range numeric entities without throwing', () => {
|
||||
expect(htmlToText('a<!-- hidden -->b')).toBe('a b');
|
||||
expect(() => htmlToText('� �')).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('normaliseText', () => {
|
||||
it('collapses runs of spaces, tabs and non-breaking spaces', () => {
|
||||
expect(normaliseText('a \t b')).toBe('a b');
|
||||
});
|
||||
it('caps blank-line runs and handles null/undefined', () => {
|
||||
expect(normaliseText('a\n\n\n\n\nb')).toBe('a\n\nb');
|
||||
expect(normaliseText(undefined)).toBe('');
|
||||
expect(normaliseText(null)).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
function baseEmail(overrides: Partial<Email> = {}): Email {
|
||||
return {
|
||||
id: 'M1', threadId: 'T1', mailboxIds: { mb1: true }, keywords: {},
|
||||
size: 100, receivedAt: '2026-08-01T10:00:00Z', hasAttachment: false,
|
||||
...overrides,
|
||||
} as Email;
|
||||
}
|
||||
|
||||
describe('emailBodyText', () => {
|
||||
it('prefers the text/plain part', () => {
|
||||
const email = baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
htmlBody: [{ partId: 'p2' } as EmailBodyPart],
|
||||
bodyValues: { p1: { value: 'plain wins' }, p2: { value: '<b>html loses</b>' } },
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('plain wins');
|
||||
});
|
||||
|
||||
it('falls back to flattened HTML when there is no plain alternative', () => {
|
||||
const email = baseEmail({
|
||||
htmlBody: [{ partId: 'p2' } as EmailBodyPart],
|
||||
bodyValues: { p2: { value: '<p>hello</p><p>world</p>' } },
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('hello\nworld');
|
||||
});
|
||||
|
||||
it('falls back to preview when bodyValues is missing entirely', () => {
|
||||
// This is the shape a caller gets when the Email/get omitted
|
||||
// fetchTextBodyValues - a silent empty body if we did not handle it.
|
||||
const email = baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
preview: 'server preview text',
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('server preview text');
|
||||
});
|
||||
|
||||
it('treats a whitespace-only plain part as absent', () => {
|
||||
const email = baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
htmlBody: [{ partId: 'p2' } as EmailBodyPart],
|
||||
bodyValues: { p1: { value: ' \n ' }, p2: { value: 'real content' } },
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('real content');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractMail', () => {
|
||||
it('flattens addresses into `people` and keeps metadata', () => {
|
||||
const doc = extractMail('acc1', baseEmail({
|
||||
subject: 'Quarterly budget',
|
||||
from: [{ name: 'Sophie Müller', email: 'sophie@example.com' }],
|
||||
to: [{ email: 'me@example.com' }],
|
||||
cc: [{ name: 'Bob', email: 'bob@example.com' }],
|
||||
preview: 'hi',
|
||||
}));
|
||||
expect(doc.contentType).toBe('mail');
|
||||
expect(doc.title).toBe('Quarterly budget');
|
||||
expect(doc.people).toContain('Sophie Müller sophie@example.com');
|
||||
expect(doc.people).toContain('bob@example.com');
|
||||
expect(doc.occurredAt).toBe('2026-08-01T10:00:00Z');
|
||||
expect(doc.metadata.threadId).toBe('T1');
|
||||
expect(doc.metadata.mailboxIds).toEqual(['mb1']);
|
||||
});
|
||||
|
||||
it('substitutes a placeholder title rather than indexing an empty one', () => {
|
||||
expect(extractMail('acc1', baseEmail()).title).toBe('(no subject)');
|
||||
});
|
||||
|
||||
it('clamps a huge body', () => {
|
||||
const doc = extractMail('acc1', baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
bodyValues: { p1: { value: 'x'.repeat(MAX_BODY_CHARS * 2) } },
|
||||
}));
|
||||
expect(doc.body.length).toBe(MAX_BODY_CHARS);
|
||||
});
|
||||
});
|
||||
|
||||
function baseEvent(overrides: Partial<CalendarEvent> = {}): CalendarEvent {
|
||||
return {
|
||||
id: 'E1', calendarIds: { c1: true }, isDraft: false, isOrigin: true,
|
||||
utcStart: '2026-08-10T09:00:00Z', utcEnd: '2026-08-10T10:00:00Z',
|
||||
'@type': 'Event', uid: 'u1', title: 'Standup', description: '',
|
||||
descriptionContentType: 'text/plain', created: null, updated: '2026-08-01T00:00:00Z',
|
||||
sequence: 0, start: '2026-08-10T11:00:00', duration: 'PT1H', timeZone: 'Europe/Zurich',
|
||||
showWithoutTime: false, status: 'confirmed', freeBusyStatus: 'busy', privacy: 'public',
|
||||
color: null, keywords: null, categories: null, locale: null, replyTo: null,
|
||||
organizerCalendarAddress: null, participants: null, mayInviteSelf: false,
|
||||
mayInviteOthers: false, hideAttendees: false, recurrenceId: null,
|
||||
recurrenceIdTimeZone: null, recurrenceRules: null, recurrenceOverrides: null,
|
||||
excludedRecurrenceRules: null, useDefaultAlerts: false, alerts: null,
|
||||
locations: null, virtualLocations: null, links: null, relatedTo: null,
|
||||
...overrides,
|
||||
} as CalendarEvent;
|
||||
}
|
||||
|
||||
describe('extractCalendarEvent', () => {
|
||||
it('indexes description, location, attendees and organizer', () => {
|
||||
const doc = extractCalendarEvent('acc1', baseEvent({
|
||||
title: 'Lease decision',
|
||||
description: 'Zurich office lease renewal',
|
||||
locations: { l1: { '@type': 'Location', name: 'Room 3.14', description: null, locationTypes: null, coordinates: null, timeZone: null, links: null, relativeTo: null } },
|
||||
organizerCalendarAddress: 'mailto:boss@example.com',
|
||||
// A partial participant on purpose: servers omit most JSCalendar fields,
|
||||
// and the extractor must cope with exactly this shape.
|
||||
participants: {
|
||||
p1: { name: 'Ana', email: 'ana@example.com', sendTo: { imip: 'mailto:ana@example.com' } } as unknown as CalendarParticipant,
|
||||
},
|
||||
}));
|
||||
expect(doc.title).toBe('Lease decision');
|
||||
expect(doc.body).toContain('Zurich office lease renewal');
|
||||
expect(doc.body).toContain('Room 3.14');
|
||||
// mailto: prefixes stripped so the address tokenises like every other one.
|
||||
expect(doc.people).toContain('boss@example.com');
|
||||
expect(doc.people).not.toContain('mailto:');
|
||||
expect(doc.people).toContain('ana@example.com');
|
||||
expect(doc.metadata.participantCount).toBe(1);
|
||||
});
|
||||
|
||||
it('flattens an HTML description', () => {
|
||||
const doc = extractCalendarEvent('acc1', baseEvent({
|
||||
description: '<p>agenda</p><script>bad()</script>',
|
||||
descriptionContentType: 'text/html',
|
||||
}));
|
||||
expect(doc.body).toContain('agenda');
|
||||
expect(doc.body).not.toContain('bad()');
|
||||
});
|
||||
|
||||
it('prefers utcStart over the zone-less local start for ordering', () => {
|
||||
expect(extractCalendarEvent('acc1', baseEvent()).occurredAt).toBe('2026-08-10T09:00:00Z');
|
||||
expect(extractCalendarEvent('acc1', baseEvent({ utcStart: null })).occurredAt)
|
||||
.toBe('2026-08-10T11:00:00');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractContact', () => {
|
||||
const card = (overrides: Partial<ContactCard> = {}): ContactCard =>
|
||||
({ id: 'C1', addressBookIds: { a1: true }, ...overrides }) as ContactCard;
|
||||
|
||||
it('uses name.full when present', () => {
|
||||
expect(contactDisplayName(card({ name: { full: 'Ada Lovelace' } }))).toBe('Ada Lovelace');
|
||||
});
|
||||
|
||||
it('assembles components in the right order when full is absent', () => {
|
||||
expect(contactDisplayName(card({
|
||||
name: { components: [{ kind: 'surname', value: 'Hopper' }, { kind: 'given', value: 'Grace' }] },
|
||||
}))).toBe('Grace Hopper');
|
||||
});
|
||||
|
||||
it('degrades to an email, then an org, then a placeholder', () => {
|
||||
expect(contactDisplayName(card({ emails: { e: { address: 'x@y.z' } } }))).toBe('x@y.z');
|
||||
expect(contactDisplayName(card({ organizations: { o: { name: 'ACME' } } }))).toBe('ACME');
|
||||
expect(contactDisplayName(card())).toBe('(unnamed contact)');
|
||||
});
|
||||
|
||||
it('puts emails and phones in `people` and notes/orgs in `body`', () => {
|
||||
const doc = extractContact('acc1', card({
|
||||
name: { full: 'Ada Lovelace' },
|
||||
emails: { e1: { address: 'ada@example.com' } },
|
||||
phones: { p1: { number: '+41 44 000 00 00' } },
|
||||
organizations: { o1: { name: 'Analytical Engines' } },
|
||||
notes: { n1: { note: 'met at the Zurich conference' } },
|
||||
nicknames: { k1: { name: 'The Countess' } },
|
||||
}));
|
||||
expect(doc.people).toContain('ada@example.com');
|
||||
expect(doc.people).toContain('+41 44 000 00 00');
|
||||
expect(doc.people).toContain('The Countess');
|
||||
expect(doc.body).toContain('Analytical Engines');
|
||||
expect(doc.body).toContain('met at the Zurich conference');
|
||||
// A contact has no single meaningful date; ranking is relevance-only.
|
||||
expect(doc.occurredAt).toBeNull();
|
||||
});
|
||||
|
||||
it('handles both RFC 9553 and legacy flat address shapes', () => {
|
||||
expect(extractContact('acc1', card({ addresses: { a: { full: 'Bahnhofstrasse 1, Zurich' } } })).body)
|
||||
.toContain('Bahnhofstrasse 1, Zurich');
|
||||
expect(extractContact('acc1', card({ addresses: { a: { street: 'Bahnhofstrasse 1', locality: 'Zurich' } } })).body)
|
||||
.toContain('Bahnhofstrasse 1, Zurich');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractFile', () => {
|
||||
const node = (overrides: Partial<FileNode> = {}): FileNode =>
|
||||
({
|
||||
id: 'F1', parentId: null, name: 'invoice.pdf', type: 'application/pdf',
|
||||
blobId: 'b1', size: 1234, created: '2026-07-01T00:00:00Z',
|
||||
modified: '2026-07-15T00:00:00Z', ...overrides,
|
||||
}) as FileNode;
|
||||
|
||||
it('indexes metadata only and says so', () => {
|
||||
const doc = extractFile('acc1', node(), { path: 'Finance/2026' });
|
||||
expect(doc.title).toBe('invoice.pdf');
|
||||
expect(doc.body).toContain('Finance/2026');
|
||||
expect(doc.body).toContain('pdf');
|
||||
expect(doc.metadata.contentIndexed).toBe(false);
|
||||
expect(doc.metadata.mimeType).toBe('application/pdf');
|
||||
expect(doc.metadata.size).toBe(1234);
|
||||
});
|
||||
|
||||
it('uses `modified` (FileNode has no `updated`) and falls back to `created`', () => {
|
||||
expect(extractFile('acc1', node()).occurredAt).toBe('2026-07-15T00:00:00Z');
|
||||
expect(extractFile('acc1', node({ modified: undefined as unknown as string })).occurredAt)
|
||||
.toBe('2026-07-01T00:00:00Z');
|
||||
});
|
||||
|
||||
it('marks directories', () => {
|
||||
const doc = extractFile('acc1', node({ name: 'Finance', type: 'd', blobId: null }));
|
||||
expect(doc.metadata.isDirectory).toBe(true);
|
||||
expect(doc.metadata.mimeType).toBeNull();
|
||||
expect(doc.body).toContain('folder');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildFilePaths', () => {
|
||||
it('resolves the PARENT chain, excluding the node itself', () => {
|
||||
const nodes = [
|
||||
{ id: 'root', parentId: null, name: 'Finance' },
|
||||
{ id: 'year', parentId: 'root', name: '2026' },
|
||||
{ id: 'file', parentId: 'year', name: 'invoice.pdf' },
|
||||
] as FileNode[];
|
||||
const paths = buildFilePaths(nodes);
|
||||
expect(paths.get('file')).toBe('Finance/2026');
|
||||
expect(paths.get('year')).toBe('Finance');
|
||||
expect(paths.get('root')).toBe('');
|
||||
});
|
||||
|
||||
it('truncates rather than failing when an ancestor is not in the set', () => {
|
||||
const nodes = [{ id: 'file', parentId: 'missing', name: 'x.txt' }] as FileNode[];
|
||||
expect(buildFilePaths(nodes).get('file')).toBe('');
|
||||
});
|
||||
|
||||
it('terminates on a parent cycle', () => {
|
||||
const nodes = [
|
||||
{ id: 'a', parentId: 'b', name: 'A' },
|
||||
{ id: 'b', parentId: 'a', name: 'B' },
|
||||
] as FileNode[];
|
||||
expect(() => buildFilePaths(nodes)).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,261 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { isSqlcipherAvailable } from '../binding';
|
||||
import { accountFileToken, getStoreDir, indexDbPath, STORE_DIR_ENV } from '../paths';
|
||||
import { MailIndex, toFtsMatchQuery, type IndexDoc } from '../store';
|
||||
|
||||
describe('toFtsMatchQuery', () => {
|
||||
it('quotes every token so FTS5 operators in user input cannot break the query', () => {
|
||||
// FTS5's MATCH grammar is NOT protected by SQL parameter binding: a bare
|
||||
// quote or a stray NEAR/AND/* raises `fts5: syntax error`, which would turn
|
||||
// a search box into a 500.
|
||||
expect(toFtsMatchQuery('a" OR b')).toBe('"a" AND "OR" AND "b"');
|
||||
// No trailing `*` here: the final token is one character, below the
|
||||
// prefix-match threshold (see the next test).
|
||||
expect(toFtsMatchQuery('NEAR(x y)')).toBe('"NEAR" AND "x" AND "y"');
|
||||
expect(toFtsMatchQuery('NEAR(x yes)')).toBe('"NEAR" AND "x" AND "yes"*');
|
||||
expect(toFtsMatchQuery('foo*')).toBe('"foo"*');
|
||||
expect(toFtsMatchQuery('a AND NOT b')).toContain('"NOT"');
|
||||
});
|
||||
|
||||
it('prefix-matches only the final token, and only when it is long enough', () => {
|
||||
expect(toFtsMatchQuery('zurich lea')).toBe('"zurich" AND "lea"*');
|
||||
// Two characters would match too much of a mailbox to be useful.
|
||||
expect(toFtsMatchQuery('zurich le')).toBe('"zurich" AND "le"');
|
||||
});
|
||||
|
||||
it('keeps unicode letters, emails and hyphenated words', () => {
|
||||
expect(toFtsMatchQuery('Müller')).toBe('"Müller"*');
|
||||
expect(toFtsMatchQuery('東京')).toBe('"東京"');
|
||||
expect(toFtsMatchQuery('a@b.com')).toBe('"a@b.com"*');
|
||||
expect(toFtsMatchQuery("O'Brien-Smith")).toBe('"O\'Brien-Smith"*');
|
||||
});
|
||||
|
||||
it('returns null for input with no usable tokens', () => {
|
||||
expect(toFtsMatchQuery('')).toBeNull();
|
||||
expect(toFtsMatchQuery(' ')).toBeNull();
|
||||
expect(toFtsMatchQuery('***')).toBeNull();
|
||||
expect(toFtsMatchQuery(undefined as unknown as string)).toBeNull();
|
||||
});
|
||||
|
||||
it('bounds the token count', () => {
|
||||
const many = Array.from({ length: 100 }, (_, i) => `w${i}`).join(' ');
|
||||
expect((toFtsMatchQuery(many) ?? '').split(' AND ')).toHaveLength(24);
|
||||
});
|
||||
});
|
||||
|
||||
describe('paths', () => {
|
||||
const original = process.env[STORE_DIR_ENV];
|
||||
afterEach(() => {
|
||||
if (original === undefined) delete process.env[STORE_DIR_ENV];
|
||||
else process.env[STORE_DIR_ENV] = original;
|
||||
});
|
||||
|
||||
it('is disabled unless the env var is set - the hosted-deployment gate', () => {
|
||||
delete process.env[STORE_DIR_ENV];
|
||||
expect(getStoreDir()).toBeNull();
|
||||
process.env[STORE_DIR_ENV] = '';
|
||||
expect(getStoreDir()).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects a relative path, which would resolve against the server cwd', () => {
|
||||
process.env[STORE_DIR_ENV] = 'offline';
|
||||
expect(getStoreDir()).toBeNull();
|
||||
process.env[STORE_DIR_ENV] = '/abs/offline';
|
||||
expect(getStoreDir()).toBe('/abs/offline');
|
||||
});
|
||||
|
||||
it('hashes the filename so the directory is not an account inventory', () => {
|
||||
const token = accountFileToken('linus@example.com');
|
||||
expect(token).toMatch(/^[0-9a-f]{32}$/);
|
||||
expect(token).not.toContain('linus');
|
||||
expect(indexDbPath('/s', 'linus@example.com')).toBe(`/s/index/${token}.db`);
|
||||
// Deterministic - the same account must resolve to the same file forever.
|
||||
expect(accountFileToken('linus@example.com')).toBe(token);
|
||||
});
|
||||
});
|
||||
|
||||
function doc(overrides: Partial<IndexDoc> = {}): IndexDoc {
|
||||
return {
|
||||
jmapAccountId: 'acc1',
|
||||
contentType: 'mail',
|
||||
id: 'M1',
|
||||
title: 'Quarterly budget review',
|
||||
people: 'Sophie Müller sophie@example.com',
|
||||
body: 'The Zurich office lease renewal needs a decision before September.',
|
||||
occurredAt: '2026-08-01T10:00:00Z',
|
||||
metadata: { threadId: 'T1' },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// The native binding is an OPTIONAL dependency, so these skip rather than fail
|
||||
// on a platform with no prebuild (e.g. Alpine/musl in CI containers).
|
||||
describe.skipIf(!isSqlcipherAvailable())('MailIndex (real SQLCipher)', () => {
|
||||
let storeDir: string;
|
||||
const accountId = 'linus@example.com';
|
||||
const key = randomBytes(32);
|
||||
|
||||
beforeEach(() => {
|
||||
storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mail-index-test-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(storeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const open = () => MailIndex.open({ storeDir, accountId, key });
|
||||
|
||||
it('writes an ENCRYPTED file - no plaintext recoverable from the raw bytes', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
index.close();
|
||||
|
||||
const bytes = fs.readFileSync(indexDbPath(storeDir, accountId));
|
||||
// The canary check, not just a header check: this is the assertion that
|
||||
// would have caught `PRAGMA key` being a silent no-op.
|
||||
expect(bytes.includes('Zurich office lease')).toBe(false);
|
||||
expect(bytes.includes('Quarterly budget')).toBe(false);
|
||||
expect(bytes.subarray(0, 15).toString('latin1')).not.toBe('SQLite format 3');
|
||||
});
|
||||
|
||||
it('rejects a wrong key and rebuilds instead of throwing at the caller', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
index.close();
|
||||
|
||||
// A different key cannot read the data; the store recreates the file rather
|
||||
// than surfacing an unrecoverable error, because the index is derived data
|
||||
// and the key was never a user secret.
|
||||
const other = MailIndex.open({ storeDir, accountId, key: randomBytes(32) });
|
||||
expect(other.search({ query: 'Zurich' })).toHaveLength(0);
|
||||
other.close();
|
||||
});
|
||||
|
||||
it('refuses a key of the wrong length', () => {
|
||||
expect(() => MailIndex.open({ storeDir, accountId, key: randomBytes(16) })).toThrow(/32 bytes/);
|
||||
});
|
||||
|
||||
it('finds documents by body, title and people', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
expect(index.search({ query: 'Zurich' }).map((h) => h.id)).toEqual(['M1']);
|
||||
expect(index.search({ query: 'quarterly' }).map((h) => h.id)).toEqual(['M1']);
|
||||
expect(index.search({ query: 'sophie@example.com' }).map((h) => h.id)).toEqual(['M1']);
|
||||
expect(index.search({ query: 'nonexistentword' })).toHaveLength(0);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('returns a snippet for use as LLM context', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
const [hit] = index.search({ query: 'Zurich' });
|
||||
expect(hit.snippet).toContain('[Zurich]');
|
||||
expect(hit.metadata.threadId).toBe('T1');
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('upserting the same id REPLACES the FTS row rather than duplicating it', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
index.upsert([doc({ body: 'Completely different content about Geneva.' })]);
|
||||
|
||||
// One row, and the OLD text must no longer match - the classic
|
||||
// stale-FTS-row bug when the index is maintained by hand.
|
||||
expect(index.search({ query: 'Geneva' })).toHaveLength(1);
|
||||
expect(index.search({ query: 'Zurich' })).toHaveLength(0);
|
||||
expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(1);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('scopes rows by JMAP account, so delegated accounts cannot merge', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ jmapAccountId: 'acc1', id: 'X', body: 'shared secret alpha' }),
|
||||
// Same JMAP id under a different account - legal, since JMAP ids are only
|
||||
// unique within an account (see namespaceMailboxIds in lib/jmap/client.ts).
|
||||
doc({ jmapAccountId: 'acc2', id: 'X', body: 'shared secret beta' }),
|
||||
]);
|
||||
expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(2);
|
||||
const hits = index.search({ query: 'secret' });
|
||||
expect(hits).toHaveLength(2);
|
||||
expect(new Set(hits.map((h) => h.jmapAccountId))).toEqual(new Set(['acc1', 'acc2']));
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('filters by content type and searches across all four by default', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ contentType: 'mail', id: 'm', title: 'Zurich mail' }),
|
||||
doc({ contentType: 'calendar', id: 'c', title: 'Zurich meeting' }),
|
||||
doc({ contentType: 'contact', id: 'k', title: 'Zurich person', occurredAt: null }),
|
||||
doc({ contentType: 'file', id: 'f', title: 'Zurich file' }),
|
||||
]);
|
||||
expect(index.search({ query: 'Zurich' })).toHaveLength(4);
|
||||
expect(index.search({ query: 'Zurich', types: ['calendar'] }).map((h) => h.id)).toEqual(['c']);
|
||||
expect(new Set(index.search({ query: 'Zurich', types: ['mail', 'file'] }).map((h) => h.id)))
|
||||
.toEqual(new Set(['m', 'f']));
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('weights a title hit above a body-only hit', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ id: 'body-only', title: 'unrelated', body: 'mentions lease once' }),
|
||||
doc({ id: 'in-title', title: 'lease renewal', body: 'unrelated text' }),
|
||||
]);
|
||||
// bm25 is negative and lower is better, so the title hit must come first.
|
||||
expect(index.search({ query: 'lease' })[0].id).toBe('in-title');
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('removes documents and their FTS rows', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
expect(index.remove('acc1', 'mail', ['M1'])).toBe(1);
|
||||
expect(index.search({ query: 'Zurich' })).toHaveLength(0);
|
||||
expect(index.remove('acc1', 'mail', ['does-not-exist'])).toBe(0);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('prunes by date without touching newer rows', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ id: 'old', occurredAt: '2020-01-01T00:00:00Z', body: 'ancient lease' }),
|
||||
doc({ id: 'new', occurredAt: '2026-08-01T00:00:00Z', body: 'current lease' }),
|
||||
]);
|
||||
expect(index.pruneOlderThan('acc1', 'mail', '2026-01-01T00:00:00Z')).toBe(1);
|
||||
expect(index.search({ query: 'lease' }).map((h) => h.id)).toEqual(['new']);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('reports existing ids and per-type stats', () => {
|
||||
const index = open();
|
||||
index.upsert([doc({ id: 'a' }), doc({ id: 'b' }), doc({ contentType: 'file', id: 'f' })]);
|
||||
expect(index.existingIds('acc1', 'mail')).toEqual(new Set(['a', 'b']));
|
||||
const stats = index.stats();
|
||||
expect(stats.find((s) => s.contentType === 'mail')?.count).toBe(2);
|
||||
expect(stats.find((s) => s.contentType === 'file')?.count).toBe(1);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('survives reopening and keeps the data', () => {
|
||||
const first = open();
|
||||
first.upsert([doc()]);
|
||||
first.close();
|
||||
const second = open();
|
||||
expect(second.search({ query: 'Zurich' })).toHaveLength(1);
|
||||
second.close();
|
||||
});
|
||||
|
||||
it('tolerates a hostile query string end to end', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
for (const q of ['"', '*', 'a" OR "b', 'NEAR(', ')', 'AND', '^', ':', '-']) {
|
||||
expect(() => index.search({ query: q })).not.toThrow();
|
||||
}
|
||||
index.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
// Guarded loader for the SQLCipher native binding.
|
||||
//
|
||||
// WHY THIS FILE EXISTS AT ALL: `@signalapp/sqlcipher` is declared in
|
||||
// package.json's `optionalDependencies`, not `dependencies`, and it MUST stay
|
||||
// there. It publishes six N-API prebuilds (darwin/linux/win32 x arm64/x64) and
|
||||
// **no build sources at all** - the published tarball has no `binding.gyp`, no
|
||||
// `src/`, no `deps/`. Its install script is `node-gyp-build`, which falls back
|
||||
// to `node-gyp rebuild` when no prebuild matches, and that fallback cannot
|
||||
// succeed without sources. So on a platform with no matching prebuild the
|
||||
// install FAILS.
|
||||
//
|
||||
// Both Dockerfiles in this repo are `FROM node:24-alpine` + `npm ci`
|
||||
// (`Dockerfile:1-4`, `integration/webmail.Dockerfile:15-19`). Alpine is musl;
|
||||
// there is no `linuxmusl-*` prebuild (and the glibc prebuild could not load
|
||||
// there anyway). As a hard `dependencies` entry this would break the
|
||||
// production image build and the integration fixture's webmail container -
|
||||
// neither of which wants this feature, they just need `npm ci` to exit 0.
|
||||
// `optionalDependencies` makes npm treat that install failure as non-fatal and
|
||||
// simply omit the package.
|
||||
//
|
||||
// The cost of that choice is exactly this module: the require must be guarded
|
||||
// at runtime, because "installed" is no longer guaranteed. Callers get
|
||||
// `null` and the feature turns itself off, which is the correct behaviour for
|
||||
// a desktop-only search index in a server that may not be a desktop.
|
||||
|
||||
/**
|
||||
* Minimal structural type for the bits of `@signalapp/sqlcipher` we use.
|
||||
*
|
||||
* Deliberately hand-written rather than `typeof import('@signalapp/sqlcipher')`:
|
||||
* the package is optional, so a type-only import would make `tsc` fail on any
|
||||
* machine where the install was skipped - which is every Alpine CI container.
|
||||
*
|
||||
* NOTE the parameter shape. `@signalapp/sqlcipher` is NOT drop-in compatible
|
||||
* with better-sqlite3 here: its `#checkParams` throws
|
||||
* `TypeError: Params must be either object or array`, so `stmt.run(a, b, c)`
|
||||
* (varargs, which better-sqlite3 accepts) is a runtime error. Always pass a
|
||||
* single array or object. Found by executing it, not by reading the types.
|
||||
*/
|
||||
export interface SqlcipherStatement {
|
||||
run(params?: readonly unknown[] | Record<string, unknown>): { changes: number; lastInsertRowid: number };
|
||||
get(params?: readonly unknown[] | Record<string, unknown>): Record<string, unknown> | undefined;
|
||||
all(params?: readonly unknown[] | Record<string, unknown>): Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface SqlcipherDatabase {
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): SqlcipherStatement;
|
||||
pragma(source: string): unknown;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface SqlcipherConstructor {
|
||||
new (path?: string): SqlcipherDatabase;
|
||||
}
|
||||
|
||||
let cached: SqlcipherConstructor | null | undefined;
|
||||
|
||||
/**
|
||||
* Returns the Database constructor, or `null` when the optional native binding
|
||||
* is not installed / cannot load on this platform. Never throws.
|
||||
*
|
||||
* Memoised on both outcomes so a missing binding costs one failed require per
|
||||
* process rather than one per request.
|
||||
*/
|
||||
export function loadSqlcipher(): SqlcipherConstructor | null {
|
||||
if (cached !== undefined) return cached;
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const mod = require('@signalapp/sqlcipher') as
|
||||
| { default?: SqlcipherConstructor }
|
||||
| SqlcipherConstructor;
|
||||
const ctor = (mod as { default?: SqlcipherConstructor }).default ?? (mod as SqlcipherConstructor);
|
||||
cached = typeof ctor === 'function' ? ctor : null;
|
||||
} catch {
|
||||
cached = null;
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
/** True when the local index can work at all in this process. */
|
||||
export function isSqlcipherAvailable(): boolean {
|
||||
return loadSqlcipher() !== null;
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
// PURE JMAP-object -> IndexDoc extractors.
|
||||
//
|
||||
// Deliberately free of database, network and store access so every shape
|
||||
// decision here is unit-testable on its own. The JMAP shapes are awkward
|
||||
// enough (JSContact keyed maps, JSCalendar participants, FileNode's `modified`
|
||||
// rather than `updated`) that this is where the bugs would otherwise hide.
|
||||
|
||||
import type { Email, CalendarEvent, ContactCard, FileNode, EmailAddress } from '@/lib/jmap/types';
|
||||
import type { IndexDoc } from './store';
|
||||
|
||||
/** Hard cap on indexed body text per document. Keeps one enormous mail from dominating the file. */
|
||||
export const MAX_BODY_CHARS = 32_000;
|
||||
|
||||
/**
|
||||
* Minimal HTML -> text, for mail that has no `text/plain` alternative.
|
||||
*
|
||||
* Not a sanitiser and not trying to be: this output is never rendered, only
|
||||
* tokenised by FTS5 and possibly handed to an LLM as context. The repo's
|
||||
* `dompurify` needs a DOM and this runs in Node, so a DOM-free reduction is the
|
||||
* right tool. Order matters - script/style content must go before tags are
|
||||
* stripped, or their contents would leak into the index as searchable text.
|
||||
*/
|
||||
export function htmlToText(html: string): string {
|
||||
return html
|
||||
.replace(/<!--[\s\S]*?-->/g, ' ')
|
||||
.replace(/<(script|style|head)\b[\s\S]*?<\/\1>/gi, ' ')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/(p|div|tr|li|h[1-6]|blockquote)>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/&/gi, '&')
|
||||
.replace(/</gi, '<')
|
||||
.replace(/>/gi, '>')
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/&#(\d+);/g, (_m, d: string) => {
|
||||
const code = Number(d);
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
|
||||
})
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_m, h: string) => {
|
||||
const code = parseInt(h, 16);
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
|
||||
})
|
||||
.replace(/[ \t\u00a0]+/g, ' ')
|
||||
.replace(/\s*\n\s*/g, '\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function normaliseText(s: string | null | undefined): string {
|
||||
if (!s) return '';
|
||||
return s.replace(/\r\n?/g, '\n').replace(/[ \t\u00a0]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
function clamp(s: string, max = MAX_BODY_CHARS): string {
|
||||
return s.length <= max ? s : s.slice(0, max);
|
||||
}
|
||||
|
||||
function formatAddresses(list: readonly EmailAddress[] | undefined): string {
|
||||
if (!list || list.length === 0) return '';
|
||||
return list
|
||||
.map((a) => [a.name, a.email].filter((p) => typeof p === 'string' && p.length > 0).join(' '))
|
||||
.filter((s) => s.length > 0)
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
/** Values of a JSContact/JSCalendar keyed map, in a stable order. */
|
||||
function mapValues<T>(m: Record<string, T> | null | undefined): T[] {
|
||||
if (!m || typeof m !== 'object') return [];
|
||||
return Object.keys(m).sort().map((k) => m[k]);
|
||||
}
|
||||
|
||||
function joinUnique(parts: Array<string | undefined | null>): string {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const p of parts) {
|
||||
const v = typeof p === 'string' ? p.trim() : '';
|
||||
if (!v || seen.has(v)) continue;
|
||||
seen.add(v);
|
||||
out.push(v);
|
||||
}
|
||||
return out.join(', ');
|
||||
}
|
||||
|
||||
// ── mail ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Resolves an Email's plain-text body from `bodyValues`, preferring the
|
||||
* `text/plain` alternative and falling back to flattening the HTML one.
|
||||
*
|
||||
* `textBody`/`htmlBody` reference parts by `partId`; the text itself only
|
||||
* arrives in `bodyValues` when the `Email/get` asked for it
|
||||
* (`fetchTextBodyValues` / `fetchHTMLBodyValues`). A caller that forgets that
|
||||
* gets an empty body rather than an error, which is exactly the kind of silent
|
||||
* hole worth naming here.
|
||||
*/
|
||||
export function emailBodyText(email: Email): string {
|
||||
const values = email.bodyValues ?? {};
|
||||
const fromParts = (parts: typeof email.textBody): string =>
|
||||
(parts ?? [])
|
||||
.map((p) => values[p.partId]?.value ?? '')
|
||||
.filter((v) => v.length > 0)
|
||||
.join('\n\n');
|
||||
|
||||
const plain = fromParts(email.textBody);
|
||||
if (plain.trim().length > 0) return normaliseText(plain);
|
||||
|
||||
const html = fromParts(email.htmlBody);
|
||||
if (html.trim().length > 0) return normaliseText(htmlToText(html));
|
||||
|
||||
// Last resort: the server-computed preview. Better than nothing for a search
|
||||
// index, and it costs no extra round trip.
|
||||
return normaliseText(email.preview);
|
||||
}
|
||||
|
||||
export function extractMail(jmapAccountId: string, email: Email): IndexDoc {
|
||||
const body = clamp(emailBodyText(email));
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'mail',
|
||||
id: email.id,
|
||||
title: normaliseText(email.subject) || '(no subject)',
|
||||
people: joinUnique([
|
||||
formatAddresses(email.from),
|
||||
formatAddresses(email.to),
|
||||
formatAddresses(email.cc),
|
||||
]),
|
||||
body,
|
||||
occurredAt: email.receivedAt ?? null,
|
||||
metadata: {
|
||||
threadId: email.threadId,
|
||||
from: email.from?.[0]?.email ?? null,
|
||||
fromName: email.from?.[0]?.name ?? null,
|
||||
hasAttachment: !!email.hasAttachment,
|
||||
size: email.size ?? null,
|
||||
mailboxIds: Object.keys(email.mailboxIds ?? {}),
|
||||
preview: normaliseText(email.preview).slice(0, 300),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── calendar ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function extractCalendarEvent(jmapAccountId: string, event: CalendarEvent): IndexDoc {
|
||||
const participants = mapValues(event.participants);
|
||||
const participantText = joinUnique(
|
||||
participants.flatMap((p) => [
|
||||
p?.name,
|
||||
p?.email,
|
||||
p?.calendarAddress?.replace(/^mailto:/i, ''),
|
||||
...Object.values(p?.sendTo ?? {}).map((v) =>
|
||||
typeof v === 'string' ? v.replace(/^mailto:/i, '') : '',
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
const locations = mapValues(event.locations)
|
||||
.map((l) => normaliseText(l?.name))
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
// `descriptionContentType` can legitimately be text/html.
|
||||
const rawDescription = normaliseText(event.description);
|
||||
const description = /html/i.test(event.descriptionContentType ?? '')
|
||||
? normaliseText(htmlToText(rawDescription))
|
||||
: rawDescription;
|
||||
|
||||
const keywords = Object.keys(event.keywords ?? {});
|
||||
const categories = Object.keys(event.categories ?? {});
|
||||
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'calendar',
|
||||
id: event.id,
|
||||
title: normaliseText(event.title) || '(untitled event)',
|
||||
people: joinUnique([event.organizerCalendarAddress?.replace(/^mailto:/i, ''), participantText]),
|
||||
body: clamp(
|
||||
[description, locations.join(', '), keywords.join(' '), categories.join(' ')]
|
||||
.filter((s) => s.length > 0)
|
||||
.join('\n\n'),
|
||||
),
|
||||
// `utcStart` is the resolved instant the app computes; `start` is local
|
||||
// wall-clock without a zone, so prefer utcStart for ordering.
|
||||
occurredAt: event.utcStart ?? event.start ?? null,
|
||||
metadata: {
|
||||
start: event.start ?? null,
|
||||
utcStart: event.utcStart ?? null,
|
||||
utcEnd: event.utcEnd ?? null,
|
||||
timeZone: event.timeZone ?? null,
|
||||
showWithoutTime: !!event.showWithoutTime,
|
||||
status: event.status ?? null,
|
||||
locations,
|
||||
calendarIds: Object.keys(event.calendarIds ?? {}),
|
||||
participantCount: participants.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── contacts ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function contactDisplayName(card: ContactCard): string {
|
||||
const full = normaliseText(card.name?.full);
|
||||
if (full) return full;
|
||||
const components = card.name?.components ?? [];
|
||||
const ordered = ['prefix', 'given', 'given2', 'additional', 'middle', 'surname', 'surname2', 'suffix'];
|
||||
const byKind = components
|
||||
.slice()
|
||||
.sort((a, b) => ordered.indexOf(a.kind) - ordered.indexOf(b.kind))
|
||||
.map((c) => c.value)
|
||||
.filter((v) => typeof v === 'string' && v.trim().length > 0)
|
||||
.join(' ');
|
||||
if (byKind.trim()) return normaliseText(byKind);
|
||||
const firstEmail = mapValues(card.emails)[0]?.address;
|
||||
if (firstEmail) return firstEmail;
|
||||
const org = mapValues(card.organizations)[0]?.name;
|
||||
return normaliseText(org) || '(unnamed contact)';
|
||||
}
|
||||
|
||||
export function extractContact(jmapAccountId: string, card: ContactCard): IndexDoc {
|
||||
const emails = mapValues(card.emails).map((e) => e.address).filter(Boolean);
|
||||
const phones = mapValues(card.phones).map((p) => p.number).filter(Boolean);
|
||||
const nicknames = mapValues(card.nicknames)
|
||||
.map((n) => n?.name)
|
||||
.filter((v): v is string => typeof v === 'string' && v.length > 0);
|
||||
const orgs = mapValues(card.organizations).map((o) => o.name).filter((v): v is string => !!v);
|
||||
const titles = mapValues(card.titles).map((t) => t.name).filter(Boolean);
|
||||
const notes = mapValues(card.notes).map((n) => n.note).filter(Boolean);
|
||||
// `full` (RFC 9553) when present, else the legacy flat fields vCard import
|
||||
// produces, else the ordered components. All three shapes occur in this type.
|
||||
const addresses = mapValues(card.addresses)
|
||||
.map((a) =>
|
||||
normaliseText(
|
||||
a?.full ||
|
||||
[a?.street, a?.locality, a?.region, a?.postcode, a?.country]
|
||||
.filter((p): p is string => typeof p === 'string' && p.length > 0)
|
||||
.join(', ') ||
|
||||
(a?.components ?? []).map((c) => c.value).join(' '),
|
||||
),
|
||||
)
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'contact',
|
||||
id: card.id,
|
||||
title: contactDisplayName(card),
|
||||
// Emails/phones go in `people` (weighted above body) because "who is
|
||||
// this / what's their number" is the dominant contact lookup.
|
||||
people: joinUnique([...emails, ...phones, ...nicknames]),
|
||||
body: clamp([...orgs, ...titles, ...addresses, ...notes].filter(Boolean).join('\n')),
|
||||
// A contact has no meaningful single date; JSContact `updated` is optional
|
||||
// and not on this repo's type, so leave it null and rank by relevance only.
|
||||
occurredAt: null,
|
||||
metadata: {
|
||||
kind: card.kind ?? null,
|
||||
emails,
|
||||
phones,
|
||||
organizations: orgs,
|
||||
addressBookIds: Object.keys(card.addressBookIds ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── files ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* METADATA ONLY - filename, path, dates, size, owner. Deliberately NOT file
|
||||
* content: extracting searchable text from arbitrary PDFs / office documents /
|
||||
* images is a materially bigger problem (per-format parsers, OCR, size limits,
|
||||
* untrusted-input parsing in a process holding the user's mail) and is a
|
||||
* separate piece of work. `path` is passed in by the caller because a FileNode
|
||||
* only knows its `parentId`; resolving the chain is the caller's job.
|
||||
*/
|
||||
export function extractFile(
|
||||
jmapAccountId: string,
|
||||
node: FileNode,
|
||||
opts: { path?: string; ownerName?: string } = {},
|
||||
): IndexDoc {
|
||||
const dirPath = normaliseText(opts.path);
|
||||
const isDirectory = node.type === 'd';
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'file',
|
||||
id: node.id,
|
||||
title: normaliseText(node.name) || '(unnamed file)',
|
||||
people: joinUnique([opts.ownerName, node.accountName]),
|
||||
// The path is genuinely searchable text ("that thing in Invoices/2026"),
|
||||
// and the extension is worth tokenising on its own.
|
||||
body: clamp(
|
||||
[dirPath, isDirectory ? 'folder' : node.type, fileExtension(node.name)]
|
||||
.filter((s) => s && s.length > 0)
|
||||
.join('\n'),
|
||||
),
|
||||
// FileNode has `modified`, NOT `updated` - asking for the wrong name
|
||||
// silently yields undefined (this repo hit that as #700).
|
||||
occurredAt: node.modified ?? node.created ?? null,
|
||||
metadata: {
|
||||
path: dirPath || null,
|
||||
mimeType: isDirectory ? null : node.type,
|
||||
isDirectory,
|
||||
size: typeof node.size === 'number' ? node.size : null,
|
||||
created: node.created ?? null,
|
||||
modified: node.modified ?? null,
|
||||
parentId: node.parentId ?? null,
|
||||
contentIndexed: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fileExtension(name: string | undefined): string {
|
||||
if (!name) return '';
|
||||
const i = name.lastIndexOf('.');
|
||||
return i > 0 && i < name.length - 1 ? name.slice(i + 1).toLowerCase() : '';
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
// A deliberately tiny server-side JMAP client, used only by the indexer.
|
||||
//
|
||||
// WHY NOT REUSE lib/jmap/client.ts: that class is a 7400-line renderer object.
|
||||
// It holds credentials in instance fields, uses `btoa`, opens EventSource /
|
||||
// WebSocket push connections, and wires itself into Zustand stores and toast
|
||||
// notifications. Importing it into an API route would drag all of that into the
|
||||
// server bundle for the sake of four method calls. The existing server-side
|
||||
// JMAP code in this repo (lib/auth/verify-jmap-auth.ts) already sets the
|
||||
// precedent: plain fetch + an Authorization header.
|
||||
//
|
||||
// Everything here is stateless - the caller supplies the auth header per call,
|
||||
// so there is no resident credential and nothing to invalidate.
|
||||
|
||||
import type { CalendarEvent, ContactCard, Email, FileNode } from '@/lib/jmap/types';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 30_000;
|
||||
|
||||
export const CAP_CORE = 'urn:ietf:params:jmap:core';
|
||||
export const CAP_MAIL = 'urn:ietf:params:jmap:mail';
|
||||
export const CAP_CALENDARS = 'urn:ietf:params:jmap:calendars';
|
||||
export const CAP_CONTACTS = 'urn:ietf:params:jmap:contacts';
|
||||
export const CAP_FILENODE = 'urn:ietf:params:jmap:filenode';
|
||||
|
||||
export class JmapIndexError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.name = 'JmapIndexError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export interface JmapSessionInfo {
|
||||
apiUrl: string;
|
||||
/** Server-confirmed authenticated login (JMAP Session.username). */
|
||||
username?: string;
|
||||
primaryAccounts: Record<string, string>;
|
||||
accounts: Record<string, { name?: string; isPersonal?: boolean; accountCapabilities?: Record<string, unknown> }>;
|
||||
capabilities: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pins a URL advertised by the session to the origin we authenticated against.
|
||||
*
|
||||
* `lib/jmap/client.ts` does the same thing in its rewriteSessionUrls() for the
|
||||
* renderer's benefit. Server-side it is a security control, not a convenience:
|
||||
* we attach the user's credentials to this URL, so a session document that
|
||||
* advertised an `apiUrl` on someone else's host would turn this into a
|
||||
* credential-leaking SSRF. Keep the path and query, take the origin from the
|
||||
* server URL we were configured with.
|
||||
*/
|
||||
function pinToServerOrigin(advertised: string, serverUrl: string): string {
|
||||
const base = new URL(serverUrl);
|
||||
let target: URL;
|
||||
try {
|
||||
target = new URL(advertised, base);
|
||||
} catch {
|
||||
throw new JmapIndexError('JMAP session advertised an unusable apiUrl');
|
||||
}
|
||||
return `${base.origin}${target.pathname}${target.search}`;
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
return await fetch(url, { ...init, signal: controller.signal, redirect: 'manual' });
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
throw new JmapIndexError('JMAP request timed out', 504);
|
||||
}
|
||||
throw new JmapIndexError(`JMAP request failed: ${String(error)}`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** Stalwart 307-redirects /.well-known/jmap to /jmap/session. */
|
||||
const MAX_REDIRECTS = 3;
|
||||
|
||||
export async function fetchJmapSession(serverUrl: string, authHeader: string): Promise<JmapSessionInfo> {
|
||||
const base = serverUrl.replace(/\/+$/, '');
|
||||
const origin = new URL(base).origin;
|
||||
let currentUrl = `${base}/.well-known/jmap`;
|
||||
let response: Response | undefined;
|
||||
|
||||
// Redirects must be followed EXPLICITLY, not with `redirect: 'follow'`: we
|
||||
// attach the user's credentials to every hop, so each one has to be checked to
|
||||
// still be on the origin we authenticated against. A blind follow would hand
|
||||
// the Authorization header to whatever host a misconfigured or hostile session
|
||||
// pointed at. Same reasoning (and same bound) as lib/auth/verify-jmap-auth.ts.
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
||||
response = await fetchWithTimeout(currentUrl, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: authHeader },
|
||||
});
|
||||
if (response.status < 300 || response.status >= 400) break;
|
||||
|
||||
const location = response.headers.get('location');
|
||||
if (!location) throw new JmapIndexError('JMAP session redirect had no Location header');
|
||||
const next = new URL(location, currentUrl);
|
||||
if (next.origin !== origin) {
|
||||
throw new JmapIndexError(
|
||||
`JMAP session redirected off-origin (${next.origin}); refusing to send credentials there`,
|
||||
);
|
||||
}
|
||||
currentUrl = next.toString();
|
||||
}
|
||||
|
||||
if (!response) throw new JmapIndexError('JMAP session fetch produced no response');
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new JmapIndexError('JMAP authentication failed', 401);
|
||||
}
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
throw new JmapIndexError('Too many redirects fetching the JMAP session');
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new JmapIndexError(`JMAP session fetch failed (${response.status})`);
|
||||
}
|
||||
const raw = (await response.json().catch(() => null)) as Record<string, unknown> | null;
|
||||
if (!raw || typeof raw.apiUrl !== 'string') {
|
||||
throw new JmapIndexError('Invalid JMAP session response');
|
||||
}
|
||||
return {
|
||||
apiUrl: pinToServerOrigin(raw.apiUrl, serverUrl),
|
||||
username: typeof raw.username === 'string' ? raw.username : undefined,
|
||||
primaryAccounts: (raw.primaryAccounts as Record<string, string>) ?? {},
|
||||
accounts: (raw.accounts as JmapSessionInfo['accounts']) ?? {},
|
||||
capabilities: (raw.capabilities as Record<string, unknown>) ?? {},
|
||||
};
|
||||
}
|
||||
|
||||
type MethodCall = [string, Record<string, unknown>, string];
|
||||
|
||||
/** Raw method-response tuple, `[name, args, callId]`. `name` is 'error' on failure. */
|
||||
type MethodResponse = [string, Record<string, unknown>, string];
|
||||
|
||||
export async function jmapRequest(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
using: readonly string[],
|
||||
methodCalls: readonly MethodCall[],
|
||||
): Promise<MethodResponse[]> {
|
||||
const response = await fetchWithTimeout(session.apiUrl, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: authHeader, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ using, methodCalls }),
|
||||
});
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new JmapIndexError('JMAP authentication failed', 401);
|
||||
}
|
||||
if (response.status === 429) {
|
||||
throw new JmapIndexError('JMAP server is rate limiting', 429);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new JmapIndexError(`JMAP request failed (${response.status})`);
|
||||
}
|
||||
const data = (await response.json().catch(() => null)) as { methodResponses?: MethodResponse[] } | null;
|
||||
if (!data || !Array.isArray(data.methodResponses)) {
|
||||
throw new JmapIndexError('Invalid JMAP response envelope');
|
||||
}
|
||||
return data.methodResponses;
|
||||
}
|
||||
|
||||
function firstResult(responses: MethodResponse[], expected: string): Record<string, unknown> | null {
|
||||
for (const [name, args] of responses) {
|
||||
if (name === expected) return args;
|
||||
// A method-level error is not fatal for an INDEX: a server that doesn't
|
||||
// support one data type should not fail the whole reindex. The caller
|
||||
// treats null as "nothing to index for this type".
|
||||
if (name === 'error') return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function idsOf(args: Record<string, unknown> | null): string[] {
|
||||
const ids = args?.ids;
|
||||
return Array.isArray(ids) ? ids.filter((v): v is string => typeof v === 'string') : [];
|
||||
}
|
||||
|
||||
function listOf<T>(args: Record<string, unknown> | null): T[] {
|
||||
const list = args?.list;
|
||||
return Array.isArray(list) ? (list as T[]) : [];
|
||||
}
|
||||
|
||||
export function accountIdFor(session: JmapSessionInfo, capability: string): string | null {
|
||||
const id = session.primaryAccounts[capability];
|
||||
return typeof id === 'string' && id.length > 0 ? id : null;
|
||||
}
|
||||
|
||||
export function hasCapability(session: JmapSessionInfo, capability: string): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(session.capabilities, capability);
|
||||
}
|
||||
|
||||
/** Per-ACCOUNT capability, mirroring client.ts's supportsFiles() (#563: a server can advertise it while an account has it revoked). */
|
||||
export function accountHasCapability(
|
||||
session: JmapSessionInfo,
|
||||
accountId: string,
|
||||
capability: string,
|
||||
): boolean {
|
||||
const account = session.accounts[accountId];
|
||||
if (!account) return false;
|
||||
if (account.accountCapabilities && Object.prototype.hasOwnProperty.call(account.accountCapabilities, capability)) {
|
||||
return true;
|
||||
}
|
||||
return account.isPersonal === false;
|
||||
}
|
||||
|
||||
// ── mail ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Properties needed to build a mail IndexDoc. Bodies come via bodyValues. */
|
||||
const EMAIL_INDEX_PROPERTIES = [
|
||||
'id', 'threadId', 'mailboxIds', 'keywords', 'size', 'receivedAt',
|
||||
'from', 'to', 'cc', 'subject', 'preview', 'hasAttachment',
|
||||
'textBody', 'htmlBody', 'bodyValues',
|
||||
] as const;
|
||||
|
||||
export async function getEmailsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
maxBodyBytes: number,
|
||||
): Promise<Email[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||
['Email/get', {
|
||||
accountId,
|
||||
ids: [...ids],
|
||||
properties: [...EMAIL_INDEX_PROPERTIES],
|
||||
// Without these two the bodyValues map comes back EMPTY and every
|
||||
// indexed body would silently fall back to `preview`.
|
||||
fetchTextBodyValues: true,
|
||||
fetchHTMLBodyValues: true,
|
||||
maxBodyValueBytes: maxBodyBytes,
|
||||
}, 'g'],
|
||||
]);
|
||||
return listOf<Email>(firstResult(responses, 'Email/get'));
|
||||
}
|
||||
|
||||
export async function queryRecentEmailIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
afterIso: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||
['Email/query', {
|
||||
accountId,
|
||||
filter: { after: afterIso },
|
||||
sort: [{ property: 'receivedAt', isAscending: false }],
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
}, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'Email/query'));
|
||||
}
|
||||
|
||||
// ── calendar ────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function getCalendarEventsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<CalendarEvent[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
||||
['CalendarEvent/get', { accountId, ids: [...ids] }, 'g'],
|
||||
]);
|
||||
return listOf<CalendarEvent>(firstResult(responses, 'CalendarEvent/get'));
|
||||
}
|
||||
|
||||
export async function queryCalendarEventIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
afterIso: string,
|
||||
beforeIso: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
||||
['CalendarEvent/query', {
|
||||
accountId,
|
||||
// LocalDateTime, per the note in lib/jmap/client.ts:307-312 - Stalwart
|
||||
// parses these without a zone suffix and ignores unparseable values.
|
||||
filter: { after: toLocalDateTime(afterIso), before: toLocalDateTime(beforeIso) },
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
}, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'CalendarEvent/query'));
|
||||
}
|
||||
|
||||
/** JSCalendar LocalDateTime: `YYYY-MM-DDTHH:MM:SS`, no zone designator. */
|
||||
function toLocalDateTime(iso: string): string {
|
||||
return iso.replace(/\.\d+/, '').replace(/Z$/, '').slice(0, 19);
|
||||
}
|
||||
|
||||
// ── contacts ────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function getContactsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<ContactCard[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
||||
['ContactCard/get', { accountId, ids: [...ids] }, 'g'],
|
||||
]);
|
||||
return listOf<ContactCard>(firstResult(responses, 'ContactCard/get'));
|
||||
}
|
||||
|
||||
export async function queryContactIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
||||
['ContactCard/query', { accountId, limit, calculateTotal: false }, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'ContactCard/query'));
|
||||
}
|
||||
|
||||
// ── files ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const FILENODE_INDEX_PROPERTIES = [
|
||||
'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified',
|
||||
] as const;
|
||||
|
||||
export async function getFilesForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<FileNode[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
||||
['FileNode/get', { accountId, ids: [...ids], properties: [...FILENODE_INDEX_PROPERTIES] }, 'g'],
|
||||
]);
|
||||
return listOf<FileNode>(firstResult(responses, 'FileNode/get'));
|
||||
}
|
||||
|
||||
export async function queryFileIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
||||
['FileNode/query', { accountId, filter: {}, limit, calculateTotal: false }, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'FileNode/query'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds `id -> "Parent/Child"` paths for the given nodes, walking `parentId`
|
||||
* upward. FileNode only knows its parent, so the caller has to assemble this;
|
||||
* unresolvable ancestors just truncate the path rather than failing.
|
||||
*/
|
||||
export function buildFilePaths(nodes: readonly FileNode[]): Map<string, string> {
|
||||
const byId = new Map(nodes.map((n) => [n.id, n]));
|
||||
const cache = new Map<string, string>();
|
||||
|
||||
const resolve = (id: string, depth: number): string => {
|
||||
if (depth > 32) return '';
|
||||
const cached = cache.get(id);
|
||||
if (cached !== undefined) return cached;
|
||||
const node = byId.get(id);
|
||||
if (!node) return '';
|
||||
const parent = node.parentId ? resolve(node.parentId, depth + 1) : '';
|
||||
const full = parent ? `${parent}/${node.name}` : node.name;
|
||||
cache.set(id, full);
|
||||
return full;
|
||||
};
|
||||
|
||||
const out = new Map<string, string>();
|
||||
for (const n of nodes) {
|
||||
// The document's own `path` metadata is its PARENT directory chain, so a
|
||||
// search for "Invoices" matches files inside it without the filename
|
||||
// being duplicated into the body.
|
||||
out.set(n.id, n.parentId ? resolve(n.parentId, 0) : '');
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Server-side client for the main process's key service (electron/key-service.ts).
|
||||
//
|
||||
// Asks for an account's index key over the inherited fd only when a job needs
|
||||
// it, and drops it as soon as the job finishes. There is deliberately no cache:
|
||||
// a resident plaintext key in a long-lived process is exactly the thing the OS
|
||||
// keychain exists to avoid, and a keychain round trip costs microseconds
|
||||
// against a job that makes network calls.
|
||||
|
||||
import net from 'node:net';
|
||||
|
||||
/** Set by electron/main.ts alongside VNCMAIL_DESKTOP_STORE_DIR. */
|
||||
export const KEY_FD_ENV = 'VNCMAIL_DESKTOP_KEY_FD';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
export type KeyErrorCode =
|
||||
| 'no-channel'
|
||||
| 'no-secure-storage'
|
||||
| 'key-io-failed'
|
||||
| 'key-unreadable'
|
||||
| 'bad-request'
|
||||
| 'timeout';
|
||||
|
||||
export class IndexKeyError extends Error {
|
||||
code: KeyErrorCode;
|
||||
constructor(code: KeyErrorCode, message: string) {
|
||||
super(message);
|
||||
this.name = 'IndexKeyError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
interface Pending {
|
||||
resolve: (value: { key?: string }) => void;
|
||||
reject: (error: Error) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Channel state lives on `globalThis`, NOT in module scope.
|
||||
*
|
||||
* A file descriptor can be adopted as a socket exactly ONCE per process: a
|
||||
* second `new net.Socket({ fd })` for an fd this process already owns throws
|
||||
* `EEXIST` from libuv's uv_pipe_open. Module scope is not once-per-process -
|
||||
* Next re-evaluates route modules (dev HMR, and separate module instances
|
||||
* across route bundles), so a module-scoped `let socket` produced exactly that
|
||||
* crash: `Could not open fd 3: Error: open EEXIST`, found by the integration
|
||||
* test rather than by reading the code.
|
||||
*
|
||||
* A Symbol key on globalThis is the one place in a Node process that survives
|
||||
* module re-evaluation, so adoption genuinely happens once.
|
||||
*/
|
||||
interface ChannelState {
|
||||
socket: net.Socket | null;
|
||||
nextId: number;
|
||||
pending: Map<number, Pending>;
|
||||
readBuffer: string;
|
||||
}
|
||||
|
||||
const STATE_KEY = Symbol.for('vncmail.mailIndex.keyChannel');
|
||||
|
||||
function state(): ChannelState {
|
||||
const holder = globalThis as unknown as Record<symbol, ChannelState | undefined>;
|
||||
const existing = holder[STATE_KEY];
|
||||
if (existing) return existing;
|
||||
const created: ChannelState = { socket: null, nextId: 1, pending: new Map(), readBuffer: '' };
|
||||
holder[STATE_KEY] = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
function failAll(s: ChannelState, error: Error): void {
|
||||
for (const [, p] of s.pending) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(error);
|
||||
}
|
||||
s.pending.clear();
|
||||
}
|
||||
|
||||
function getSocket(): net.Socket {
|
||||
const s = state();
|
||||
if (s.socket && !s.socket.destroyed) return s.socket;
|
||||
|
||||
const raw = process.env[KEY_FD_ENV]?.trim();
|
||||
const fd = raw ? Number(raw) : NaN;
|
||||
if (!Number.isInteger(fd) || fd < 3) {
|
||||
throw new IndexKeyError(
|
||||
'no-channel',
|
||||
`${KEY_FD_ENV} is not a usable file descriptor (got ${JSON.stringify(raw)}). ` +
|
||||
`The local index only works inside the Electron desktop shell.`,
|
||||
);
|
||||
}
|
||||
|
||||
let created: net.Socket;
|
||||
try {
|
||||
created = new net.Socket({ fd, readable: true, writable: true });
|
||||
} catch (error) {
|
||||
throw new IndexKeyError('no-channel', `Could not open fd ${fd}: ${String(error)}`);
|
||||
}
|
||||
// The channel outlives every individual request; don't let it hold the event
|
||||
// loop open on its own.
|
||||
created.unref();
|
||||
|
||||
created.on('data', (chunk: Buffer) => {
|
||||
s.readBuffer += chunk.toString('utf8');
|
||||
if (s.readBuffer.length > 64 * 1024) s.readBuffer = '';
|
||||
let newline: number;
|
||||
while ((newline = s.readBuffer.indexOf('\n')) >= 0) {
|
||||
const line = s.readBuffer.slice(0, newline);
|
||||
s.readBuffer = s.readBuffer.slice(newline + 1);
|
||||
if (!line.trim()) continue;
|
||||
let msg: { id?: unknown; ok?: unknown; key?: unknown; code?: unknown; error?: unknown };
|
||||
try {
|
||||
msg = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const id = typeof msg.id === 'number' ? msg.id : null;
|
||||
if (id === null) continue;
|
||||
const p = s.pending.get(id);
|
||||
if (!p) continue;
|
||||
s.pending.delete(id);
|
||||
clearTimeout(p.timer);
|
||||
if (msg.ok === true) {
|
||||
p.resolve({ key: typeof msg.key === 'string' ? msg.key : undefined });
|
||||
} else {
|
||||
const code = typeof msg.code === 'string' ? (msg.code as KeyErrorCode) : 'key-io-failed';
|
||||
p.reject(new IndexKeyError(code, typeof msg.error === 'string' ? msg.error : 'Key request failed'));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const onGone = (error?: Error) => {
|
||||
s.socket = null;
|
||||
s.readBuffer = '';
|
||||
failAll(s, error ?? new IndexKeyError('no-channel', 'Key service channel closed'));
|
||||
};
|
||||
created.on('close', () => onGone());
|
||||
created.on('error', (error) => onGone(new IndexKeyError('no-channel', String(error))));
|
||||
|
||||
s.socket = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
function request(op: 'getIndexKey' | 'deleteIndexKey', accountId: string): Promise<{ key?: string }> {
|
||||
const sock = getSocket();
|
||||
const s = state();
|
||||
const id = s.nextId++;
|
||||
return new Promise<{ key?: string }>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
s.pending.delete(id);
|
||||
reject(new IndexKeyError('timeout', `Key service did not answer within ${REQUEST_TIMEOUT_MS}ms`));
|
||||
}, REQUEST_TIMEOUT_MS);
|
||||
// Don't let a pending key request keep the process alive either.
|
||||
timer.unref?.();
|
||||
s.pending.set(id, { resolve, reject, timer });
|
||||
try {
|
||||
sock.write(`${JSON.stringify({ id, op, accountId })}\n`);
|
||||
} catch (error) {
|
||||
s.pending.delete(id);
|
||||
clearTimeout(timer);
|
||||
reject(new IndexKeyError('no-channel', `Could not write to the key service: ${String(error)}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `fn` with the account's raw index key, then zeroes the buffer.
|
||||
*
|
||||
* Zeroing a Buffer is genuine (unlike a JS string, which cannot be scrubbed) -
|
||||
* which is why the key crosses the boundary as hex and is converted to a Buffer
|
||||
* exactly once, here. `store.ts` puts the hex into a `PRAGMA` string, so a copy
|
||||
* does briefly exist in the JS heap; the buffer wipe bounds how long the
|
||||
* long-lived copy lives, it does not pretend to eliminate every trace.
|
||||
*/
|
||||
export async function withIndexKey<T>(
|
||||
accountId: string,
|
||||
fn: (key: Buffer) => Promise<T> | T,
|
||||
): Promise<T> {
|
||||
const { key: hex } = await request('getIndexKey', accountId);
|
||||
if (!hex) throw new IndexKeyError('key-io-failed', 'Key service returned no key');
|
||||
const key = Buffer.from(hex, 'hex');
|
||||
if (key.length !== 32) {
|
||||
key.fill(0);
|
||||
throw new IndexKeyError('key-io-failed', `Key service returned ${key.length} bytes, expected 32`);
|
||||
}
|
||||
try {
|
||||
return await fn(key);
|
||||
} finally {
|
||||
key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
/** Used when purging an account: the key goes FIRST, so an interrupted purge leaves unreadable data. */
|
||||
export async function deleteIndexKey(accountId: string): Promise<void> {
|
||||
await request('deleteIndexKey', accountId);
|
||||
}
|
||||
|
||||
/** True when this process has a key channel at all (i.e. is the desktop shell's server). */
|
||||
export function hasKeyChannel(): boolean {
|
||||
const raw = process.env[KEY_FD_ENV]?.trim();
|
||||
const fd = raw ? Number(raw) : NaN;
|
||||
return Number.isInteger(fd) && fd >= 3;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// The hosted-deployment gate, and where an account's index file lives.
|
||||
//
|
||||
// The standalone Next.js server in `electron/main.ts` is the SAME artifact the
|
||||
// production `Dockerfile` ships to multi-tenant deployments. An index that
|
||||
// activated unconditionally would have a shared server start writing every
|
||||
// user's mail into a server-side SQLite file. So activation is keyed on an env
|
||||
// var that ONLY `electron/main.ts` sets, and that same var supplies the path -
|
||||
// one variable doing both jobs, so they cannot drift apart.
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
|
||||
/** Set by electron/main.ts on spawn. Absent => the feature does not exist. */
|
||||
export const STORE_DIR_ENV = 'VNCMAIL_DESKTOP_STORE_DIR';
|
||||
|
||||
/**
|
||||
* The index root, or `null` when this process is not the desktop shell's
|
||||
* server. Every route must 404 on `null` - not 403, since nothing should learn
|
||||
* the routes exist in a deployment that doesn't have the feature.
|
||||
*/
|
||||
export function getStoreDir(): string | null {
|
||||
const dir = process.env[STORE_DIR_ENV]?.trim();
|
||||
if (!dir) return null;
|
||||
// Must be absolute: a relative path would resolve against the server's cwd,
|
||||
// which differs between `electron:dev` and a packaged build.
|
||||
if (!path.isAbsolute(dir)) return null;
|
||||
return dir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filenames are a hash, not `username@host`, so a directory listing is not a
|
||||
* plaintext inventory of the user's accounts. The account id itself lives only
|
||||
* inside the encrypted file (and in the renderer's own `account-registry`,
|
||||
* which already stores it in plain localStorage).
|
||||
*/
|
||||
export function accountFileToken(accountId: string): string {
|
||||
return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
export function indexDbPath(storeDir: string, accountId: string): string {
|
||||
return path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`);
|
||||
}
|
||||
|
||||
export function keyFilePath(storeDir: string, accountId: string): string {
|
||||
return path.join(storeDir, 'keys', `${accountFileToken(accountId)}.bin`);
|
||||
}
|
||||
|
||||
/** WAL siblings must be removed with the database, or a purge leaks readable pages. */
|
||||
export function dbSiblings(dbPath: string): string[] {
|
||||
return [dbPath, `${dbPath}-wal`, `${dbPath}-shm`];
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
// The index jobs.
|
||||
//
|
||||
// TWO SHAPES, both plain request-scoped work - there is no background worker,
|
||||
// no cursor, no retry ladder and no resident credential anywhere:
|
||||
//
|
||||
// 1. `indexDocuments()` - the PRIMARY path. The renderer's live JMAP push
|
||||
// connection sees a StateChange, and calls the route with the ids that
|
||||
// changed (or with no ids, meaning "refetch what's recent for this type").
|
||||
// One or a handful of objects, fetched and upserted.
|
||||
// 2. `catchUpAll()` - the FALLBACK. On app launch, backfill a bounded recent
|
||||
// window for every supported type, because anything that changed while the
|
||||
// app was closed produced no push event.
|
||||
//
|
||||
// Staleness between refreshes is acceptable by design: this is a search index
|
||||
// for a retrieval/AI feature, not a mail replica.
|
||||
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { generateAccountId } from '@/lib/account-utils';
|
||||
import { getStalwartCredentials } from '@/lib/stalwart/credentials';
|
||||
import { logger } from '@/lib/logger';
|
||||
import {
|
||||
accountHasCapability, accountIdFor, buildFilePaths, CAP_CALENDARS, CAP_CONTACTS,
|
||||
CAP_FILENODE, CAP_MAIL, fetchJmapSession, getCalendarEventsForIndex, getContactsForIndex,
|
||||
getEmailsForIndex, getFilesForIndex, hasCapability, JmapIndexError, queryCalendarEventIds,
|
||||
queryContactIds, queryFileIds, queryRecentEmailIds, type JmapSessionInfo,
|
||||
} from './jmap';
|
||||
import { extractCalendarEvent, extractContact, extractFile, extractMail } from './extract';
|
||||
import { withIndexKey } from './key';
|
||||
import { getStoreDir } from './paths';
|
||||
import { MailIndex, type ContentType, type IndexDoc } from './store';
|
||||
|
||||
/**
|
||||
* Bounded window. Small on purpose: this is the first cut of a retrieval index,
|
||||
* and a wide window turns "index on every delivery" into a slow request. The
|
||||
* event-driven path indexes single objects, so the window only bounds catch-up.
|
||||
*/
|
||||
export const INDEX_WINDOW_DAYS = 30;
|
||||
/** Calendar looks forward as well as back - upcoming events are the useful ones. */
|
||||
export const CALENDAR_FORWARD_DAYS = 180;
|
||||
/** Per-type ceiling for one catch-up pass. */
|
||||
export const CATCHUP_MAX_PER_TYPE = 500;
|
||||
/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
|
||||
export const MAX_IDS_PER_CALL = 200;
|
||||
/** Cap on body bytes requested per message from the server. */
|
||||
export const MAX_BODY_VALUE_BYTES = 256_000;
|
||||
/** Contacts and files have no useful date filter, so they are simply capped. */
|
||||
export const CONTACTS_MAX = 2_000;
|
||||
export const FILES_MAX = 2_000;
|
||||
|
||||
export interface IndexSession {
|
||||
serverUrl: string;
|
||||
authHeader: string;
|
||||
username: string;
|
||||
slot: number;
|
||||
/** `username@host` - the durable per-account key. NEVER the cookie slot. */
|
||||
accountId: string;
|
||||
}
|
||||
|
||||
export class IndexSessionError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status: number) {
|
||||
super(message);
|
||||
this.name = 'IndexSessionError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the calling request to an account and a usable Authorization header.
|
||||
*
|
||||
* Uses the SAME per-slot encrypted `jmap_stalwart_ctx` cookie that
|
||||
* `/api/settings`, `/api/push/preview` and `/api/plugin-approval-status`
|
||||
* already read (`lib/stalwart/credentials.ts`). That cookie is written by
|
||||
* `/api/auth/stalwart-context`, which the renderer syncs on every login,
|
||||
* session restore, SSO callback, account switch and token refresh
|
||||
* (`stores/auth-store.ts`, 10 call sites), and it carries a ready-made header
|
||||
* for BOTH basic and bearer accounts.
|
||||
*
|
||||
* Why this matters beyond convenience: it means the indexer never touches the
|
||||
* OAuth refresh-token cookie. A server-side refresh would rotate the token into
|
||||
* a response nobody reads while the browser kept the superseded one, and the
|
||||
* next real refresh would then fail and log the user out. Reading an
|
||||
* already-minted header cannot cause that.
|
||||
*/
|
||||
export async function resolveIndexSession(request: NextRequest): Promise<IndexSession> {
|
||||
const credentials = await getStalwartCredentials(request);
|
||||
if (!credentials) {
|
||||
throw new IndexSessionError('No JMAP auth context for this account; sign in again.', 401);
|
||||
}
|
||||
const accountId = generateAccountId(credentials.username, credentials.serverUrl);
|
||||
return { ...credentials, accountId };
|
||||
}
|
||||
|
||||
export interface IndexResult {
|
||||
accountId: string;
|
||||
/** Per-type counts of documents written. */
|
||||
written: Partial<Record<ContentType, number>>;
|
||||
/** Types the server (or this account) doesn't support, so nothing was attempted. */
|
||||
skipped: ContentType[];
|
||||
/** Non-fatal per-type failures. One broken type must not fail the whole call. */
|
||||
errors: Array<{ contentType: ContentType; message: string }>;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
function isoDaysFromNow(days: number): string {
|
||||
return new Date(Date.now() + days * 24 * 60 * 60 * 1000).toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Which types this session can actually index. Calendar/contacts are session
|
||||
* capabilities; files is a PER-ACCOUNT capability (a server can advertise
|
||||
* filenode while a specific account has it revoked - #563).
|
||||
*/
|
||||
export function supportedTypes(session: JmapSessionInfo): {
|
||||
supported: ContentType[];
|
||||
skipped: ContentType[];
|
||||
accountIds: Partial<Record<ContentType, string>>;
|
||||
} {
|
||||
const supported: ContentType[] = [];
|
||||
const skipped: ContentType[] = [];
|
||||
const accountIds: Partial<Record<ContentType, string>> = {};
|
||||
|
||||
const mailAccount = accountIdFor(session, CAP_MAIL);
|
||||
if (mailAccount) { supported.push('mail'); accountIds.mail = mailAccount; }
|
||||
else skipped.push('mail');
|
||||
|
||||
const calAccount = accountIdFor(session, CAP_CALENDARS);
|
||||
if (calAccount && hasCapability(session, CAP_CALENDARS)) {
|
||||
supported.push('calendar'); accountIds.calendar = calAccount;
|
||||
} else skipped.push('calendar');
|
||||
|
||||
const contactAccount = accountIdFor(session, CAP_CONTACTS);
|
||||
if (contactAccount && hasCapability(session, CAP_CONTACTS)) {
|
||||
supported.push('contact'); accountIds.contact = contactAccount;
|
||||
} else skipped.push('contact');
|
||||
|
||||
// Files fall back to the mail account id: Stalwart exposes FileNode on the
|
||||
// same account and does not always list a primaryAccounts entry for it.
|
||||
const fileAccount = accountIdFor(session, CAP_FILENODE) ?? mailAccount;
|
||||
if (fileAccount && accountHasCapability(session, fileAccount, CAP_FILENODE)) {
|
||||
supported.push('file'); accountIds.file = fileAccount;
|
||||
} else skipped.push('file');
|
||||
|
||||
return { supported, skipped, accountIds };
|
||||
}
|
||||
|
||||
interface FetchArgs {
|
||||
session: JmapSessionInfo;
|
||||
authHeader: string;
|
||||
jmapAccountId: string;
|
||||
ids: readonly string[] | null;
|
||||
}
|
||||
|
||||
/** Fetches and flattens one content type. `ids === null` means "the recent window". */
|
||||
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<IndexDoc[]> {
|
||||
const { session, authHeader, jmapAccountId, ids } = args;
|
||||
|
||||
switch (contentType) {
|
||||
case 'mail': {
|
||||
const targetIds = ids ?? await queryRecentEmailIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
// Chunked because bodies are big: one Email/get for 500 messages with
|
||||
// full bodies would be an enormous response.
|
||||
for (let i = 0; i < targetIds.length; i += 25) {
|
||||
const emails = await getEmailsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 25), MAX_BODY_VALUE_BYTES,
|
||||
);
|
||||
for (const email of emails) docs.push(extractMail(jmapAccountId, email));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'calendar': {
|
||||
const targetIds = ids ?? await queryCalendarEventIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||
CATCHUP_MAX_PER_TYPE,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 50) {
|
||||
const events = await getCalendarEventsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 50),
|
||||
);
|
||||
for (const event of events) docs.push(extractCalendarEvent(jmapAccountId, event));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'contact': {
|
||||
const targetIds = ids ?? await queryContactIds(session, authHeader, jmapAccountId, CONTACTS_MAX);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 100) {
|
||||
const cards = await getContactsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
|
||||
);
|
||||
for (const card of cards) docs.push(extractContact(jmapAccountId, card));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'file': {
|
||||
const targetIds = ids ?? await queryFileIds(session, authHeader, jmapAccountId, FILES_MAX);
|
||||
const nodes = [];
|
||||
for (let i = 0; i < targetIds.length; i += 100) {
|
||||
nodes.push(...await getFilesForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
|
||||
));
|
||||
}
|
||||
// Paths need the whole set in hand, so this one can't stream per chunk.
|
||||
const paths = buildFilePaths(nodes);
|
||||
return nodes
|
||||
// Directories are indexed too: "what's in the Invoices folder" is a
|
||||
// real query, and a folder row is a few bytes.
|
||||
.map((node) => extractFile(jmapAccountId, node, { path: paths.get(node.id) }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface IndexRequest {
|
||||
/** Types to touch. Empty means every supported type. */
|
||||
types?: readonly ContentType[];
|
||||
/**
|
||||
* Per-type ids to index. Omitted/empty for a type means "refetch that type's
|
||||
* recent window" (the catch-up shape).
|
||||
*/
|
||||
ids?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Per-type ids to REMOVE (a JMAP `destroyed`). */
|
||||
removed?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Drop documents outside the retention window after writing. */
|
||||
prune?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one index pass. Opens the encrypted store, fetches, upserts, closes.
|
||||
*
|
||||
* The key is fetched from the main process for the duration of this call only
|
||||
* (`withIndexKey`) and zeroed afterwards - there is no cached handle and no
|
||||
* resident key.
|
||||
*/
|
||||
export async function runIndex(
|
||||
indexSession: IndexSession,
|
||||
req: IndexRequest,
|
||||
): Promise<IndexResult> {
|
||||
const started = Date.now();
|
||||
const storeDir = getStoreDir();
|
||||
if (!storeDir) {
|
||||
throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
|
||||
}
|
||||
|
||||
const session = await fetchJmapSession(indexSession.serverUrl, indexSession.authHeader);
|
||||
|
||||
// Identity cross-check. `generateAccountId` used the username from the auth
|
||||
// context cookie; the server may canonicalise a short login (`linus`) to a
|
||||
// full address (`linus@example.com`) - which is exactly why AccountEntry
|
||||
// carries `serverIdentifiers`. Accept either form, reject anything else
|
||||
// rather than writing one account's mail into another's file.
|
||||
if (session.username) {
|
||||
const serverAccountId = generateAccountId(session.username, indexSession.serverUrl);
|
||||
if (serverAccountId !== indexSession.accountId) {
|
||||
const shortMatches = session.username.split('@')[0] === indexSession.username.split('@')[0];
|
||||
if (!shortMatches) {
|
||||
throw new IndexSessionError(
|
||||
'The JMAP session belongs to a different account than the request cookie.',
|
||||
409,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { supported, skipped, accountIds } = supportedTypes(session);
|
||||
const requested = req.types && req.types.length > 0 ? req.types : supported;
|
||||
const types = requested.filter((t) => supported.includes(t));
|
||||
const notAttempted = [...new Set([...skipped, ...requested.filter((t) => !supported.includes(t))])];
|
||||
|
||||
const written: Partial<Record<ContentType, number>> = {};
|
||||
const errors: IndexResult['errors'] = [];
|
||||
|
||||
await withIndexKey(indexSession.accountId, async (key) => {
|
||||
const index = MailIndex.open({ storeDir, accountId: indexSession.accountId, key });
|
||||
try {
|
||||
for (const contentType of types) {
|
||||
const jmapAccountId = accountIds[contentType];
|
||||
if (!jmapAccountId) continue;
|
||||
try {
|
||||
const removed = req.removed?.[contentType];
|
||||
if (removed && removed.length > 0) {
|
||||
index.remove(jmapAccountId, contentType, removed.slice(0, MAX_IDS_PER_CALL));
|
||||
}
|
||||
|
||||
const requestedIds = req.ids?.[contentType];
|
||||
const ids = requestedIds && requestedIds.length > 0
|
||||
? requestedIds.slice(0, MAX_IDS_PER_CALL)
|
||||
: null;
|
||||
|
||||
const docs = await fetchDocs(contentType, {
|
||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids,
|
||||
});
|
||||
written[contentType] = index.upsert(docs);
|
||||
|
||||
if (req.prune && contentType === 'mail') {
|
||||
// Only mail prunes by date: calendar's window looks forward,
|
||||
// contacts have no date, and file rows are metadata-sized.
|
||||
index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS));
|
||||
}
|
||||
} catch (error) {
|
||||
// One unsupported or misbehaving type must not fail the others.
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push({ contentType, message });
|
||||
if (error instanceof JmapIndexError && error.status === 401) throw error;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
index.close();
|
||||
}
|
||||
});
|
||||
|
||||
const result: IndexResult = {
|
||||
accountId: indexSession.accountId,
|
||||
written,
|
||||
skipped: notAttempted,
|
||||
errors,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
logger.info('mail-index: pass complete', {
|
||||
slot: indexSession.slot,
|
||||
written: JSON.stringify(written),
|
||||
skipped: notAttempted.join(',') || 'none',
|
||||
errors: errors.length,
|
||||
durationMs: result.durationMs,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,456 @@
|
||||
// The encrypted local search index: schema, open/close, upsert, search.
|
||||
//
|
||||
// One SQLite (SQLCipher) file per account. Rows are ALSO account-scoped
|
||||
// internally - `(jmap_account_id, content_type, id)` - because a single login
|
||||
// exposes the user's own JMAP account plus every delegated/shared account, and
|
||||
// JMAP ids are unique only WITHIN an account (this codebase already works
|
||||
// around that collision in `lib/jmap/client.ts:388`'s namespaceMailboxIds).
|
||||
// One file per account keeps purge trivial; the composite key keeps
|
||||
// delegated accounts from merging inside it.
|
||||
//
|
||||
// This is a SEARCH INDEX, not a mail replica. It is allowed to be stale, it is
|
||||
// allowed to be incomplete, and it can be discarded and rebuilt at any time -
|
||||
// which is why the schema-version mismatch path below simply drops everything
|
||||
// rather than migrating.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { loadSqlcipher, type SqlcipherDatabase } from './binding';
|
||||
import { dbSiblings, indexDbPath } from './paths';
|
||||
|
||||
export const SCHEMA_VERSION = 1;
|
||||
|
||||
export type ContentType = 'mail' | 'calendar' | 'contact' | 'file';
|
||||
|
||||
export const CONTENT_TYPES: readonly ContentType[] = ['mail', 'calendar', 'contact', 'file'];
|
||||
|
||||
export function isContentType(v: unknown): v is ContentType {
|
||||
return typeof v === 'string' && (CONTENT_TYPES as readonly string[]).includes(v);
|
||||
}
|
||||
|
||||
/**
|
||||
* One indexable thing, already flattened to text. Produced by the pure
|
||||
* extractors in `extract.ts` so that every JMAP-shape decision is unit-testable
|
||||
* without a database or a server.
|
||||
*/
|
||||
export interface IndexDoc {
|
||||
jmapAccountId: string;
|
||||
contentType: ContentType;
|
||||
/** JMAP id. Unique only within (jmapAccountId, contentType). */
|
||||
id: string;
|
||||
/** Subject / event title / contact display name / filename. */
|
||||
title: string;
|
||||
/** Addresses and names: sender+recipients, attendees, contact emails/phones, owner. */
|
||||
people: string;
|
||||
/** The bulk searchable text. Plain text only - never HTML. */
|
||||
body: string;
|
||||
/** ISO 8601, or null when the type has no meaningful date. Drives recency ordering. */
|
||||
occurredAt: string | null;
|
||||
/** Small type-specific extras returned verbatim to the caller (never searched). */
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface SearchHit {
|
||||
contentType: ContentType;
|
||||
id: string;
|
||||
jmapAccountId: string;
|
||||
title: string;
|
||||
people: string;
|
||||
occurredAt: string | null;
|
||||
metadata: Record<string, unknown>;
|
||||
/** FTS5 bm25 score. Lower is a better match (bm25 returns negative values). */
|
||||
score: number;
|
||||
/** Highlighted excerpt from the body, for feeding an LLM as context. */
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
const DDL = `
|
||||
CREATE TABLE IF NOT EXISTS doc (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
people TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
occurred_at TEXT,
|
||||
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||
indexed_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (jmap_account_id, content_type, id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS doc_recent
|
||||
ON doc(jmap_account_id, content_type, occurred_at DESC);
|
||||
|
||||
-- Standalone (not external-content) FTS5: the text is duplicated into this
|
||||
-- table and kept in step manually on upsert. External content would avoid the
|
||||
-- duplication but requires deleting the old FTS row using its OLD column
|
||||
-- values, which an upsert does not have to hand - a well-known source of
|
||||
-- silently-stale FTS rows. At this scale (a bounded recent window) the
|
||||
-- duplication is the cheaper correctness trade.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5(
|
||||
title, people, body,
|
||||
tokenize='unicode61 remove_diacritics 2'
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);
|
||||
`;
|
||||
|
||||
export class MailIndexUnavailableError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'MailIndexUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that the file we just opened is REALLY encrypted.
|
||||
*
|
||||
* This is not defensive boilerplate, it guards the sharpest landmine found
|
||||
* while designing this: on both `node:sqlite` and plain `better-sqlite3`,
|
||||
* `PRAGMA key = ...` is **silently accepted and does nothing** - no error, a
|
||||
* working database, and the mail sitting on disk in cleartext. Verified by
|
||||
* writing a file and recovering a canary string from the raw bytes.
|
||||
*
|
||||
* The check is on the VALUE, not the row count: a non-cipher binding returns
|
||||
* ZERO ROWS for `PRAGMA cipher_version`, so a naive `!== ''` comparison over a
|
||||
* missing row passes vacuously. Require a non-empty string.
|
||||
*/
|
||||
function assertEncrypted(db: SqlcipherDatabase, dbPath: string): void {
|
||||
const rows = db.pragma('cipher_version');
|
||||
const value =
|
||||
Array.isArray(rows) && rows.length > 0 && rows[0] && typeof rows[0] === 'object'
|
||||
? (rows[0] as Record<string, unknown>).cipher_version
|
||||
: undefined;
|
||||
if (typeof value !== 'string' || value.trim().length === 0) {
|
||||
db.close();
|
||||
throw new MailIndexUnavailableError(
|
||||
`Refusing to use ${path.basename(dbPath)}: the SQLite binding reports no SQLCipher ` +
|
||||
`support (PRAGMA cipher_version returned ${JSON.stringify(rows)}), so the index ` +
|
||||
`would be written in cleartext.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface OpenOptions {
|
||||
storeDir: string;
|
||||
accountId: string;
|
||||
/** Raw 32-byte key. Used as SQLCipher's raw key (no KDF) via `PRAGMA key = "x'..'"`. */
|
||||
key: Buffer;
|
||||
}
|
||||
|
||||
export class MailIndex {
|
||||
private constructor(
|
||||
private readonly db: SqlcipherDatabase,
|
||||
readonly dbPath: string,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Opens (creating if needed) the account's index. Throws
|
||||
* MailIndexUnavailableError when the native binding is absent or the file is
|
||||
* not actually encrypted; the caller turns the feature off rather than
|
||||
* falling back to something unencrypted.
|
||||
*/
|
||||
static open({ storeDir, accountId, key }: OpenOptions): MailIndex {
|
||||
const Database = loadSqlcipher();
|
||||
if (!Database) {
|
||||
throw new MailIndexUnavailableError(
|
||||
'@signalapp/sqlcipher is not installed for this platform (it is an optional dependency).',
|
||||
);
|
||||
}
|
||||
if (key.length !== 32) {
|
||||
throw new MailIndexUnavailableError(`Index key must be 32 bytes, got ${key.length}.`);
|
||||
}
|
||||
|
||||
const dbPath = indexDbPath(storeDir, accountId);
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: 0o700 });
|
||||
|
||||
let db = new Database(dbPath);
|
||||
// The key pragma must be the FIRST statement on the connection. Hex form
|
||||
// means SQLCipher uses these 32 bytes as the raw key with no KDF, which is
|
||||
// right for a random key (a passphrase would want the KDF).
|
||||
db.pragma(`key = "x'${key.toString('hex')}'"`);
|
||||
assertEncrypted(db, dbPath);
|
||||
|
||||
// A wrong key surfaces here rather than at open: SQLCipher only reads the
|
||||
// header lazily. Treat it as "unreadable" and rebuild from scratch - the
|
||||
// index is derived data, so there is nothing to recover and never anything
|
||||
// to prompt the user for (the key was never a user secret).
|
||||
let version: number | null;
|
||||
try {
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
// The offline replica (lib/offline-replica/**) is a SECOND connection to
|
||||
// this same file, writing disjoint tables. WAL lets a writer and readers
|
||||
// coexist, but two WRITERS get SQLITE_BUSY immediately without this - and
|
||||
// both subsystems are driven by the same renderer push handler, so they
|
||||
// genuinely do overlap.
|
||||
db.pragma('busy_timeout = 8000');
|
||||
version = readSchemaVersion(db);
|
||||
} catch {
|
||||
db.close();
|
||||
for (const f of dbSiblings(dbPath)) {
|
||||
try { fs.rmSync(f, { force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
db = new Database(dbPath);
|
||||
db.pragma(`key = "x'${key.toString('hex')}'"`);
|
||||
assertEncrypted(db, dbPath);
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
// The offline replica (lib/offline-replica/**) is a SECOND connection to
|
||||
// this same file, writing disjoint tables. WAL lets a writer and readers
|
||||
// coexist, but two WRITERS get SQLITE_BUSY immediately without this - and
|
||||
// both subsystems are driven by the same renderer push handler, so they
|
||||
// genuinely do overlap.
|
||||
db.pragma('busy_timeout = 8000');
|
||||
version = null;
|
||||
}
|
||||
|
||||
if (version !== null && version !== SCHEMA_VERSION) {
|
||||
// Rebuildable derived data: drop, don't migrate.
|
||||
db.exec('DROP TABLE IF EXISTS doc_fts; DROP TABLE IF EXISTS doc; DROP TABLE IF EXISTS meta;');
|
||||
version = null;
|
||||
}
|
||||
if (version === null) {
|
||||
db.exec(DDL);
|
||||
db.prepare('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)').run([
|
||||
'schema_version',
|
||||
String(SCHEMA_VERSION),
|
||||
]);
|
||||
}
|
||||
|
||||
return new MailIndex(db, dbPath);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.db.close(); } catch { /* already closed */ }
|
||||
}
|
||||
|
||||
/**
|
||||
* Upserts documents and keeps the FTS rows in step. Returns the number of
|
||||
* rows written. One transaction for the whole batch - a partially-applied
|
||||
* batch is harmless (it is an index) but a transaction is faster.
|
||||
*/
|
||||
upsert(docs: readonly IndexDoc[]): number {
|
||||
if (docs.length === 0) return 0;
|
||||
|
||||
const upsertDoc = this.db.prepare(`
|
||||
INSERT INTO doc (jmap_account_id, content_type, id, title, people, body,
|
||||
occurred_at, metadata_json, indexed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(jmap_account_id, content_type, id) DO UPDATE SET
|
||||
title = excluded.title, people = excluded.people, body = excluded.body,
|
||||
occurred_at = excluded.occurred_at, metadata_json = excluded.metadata_json,
|
||||
indexed_at = excluded.indexed_at
|
||||
RETURNING rowid
|
||||
`);
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const insertFts = this.db.prepare(
|
||||
'INSERT INTO doc_fts (rowid, title, people, body) VALUES (?, ?, ?, ?)',
|
||||
);
|
||||
|
||||
const now = Date.now();
|
||||
let written = 0;
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const d of docs) {
|
||||
const row = upsertDoc.get([
|
||||
d.jmapAccountId, d.contentType, d.id,
|
||||
d.title, d.people, d.body,
|
||||
d.occurredAt, JSON.stringify(d.metadata ?? {}), now,
|
||||
]);
|
||||
const rowid = row?.rowid;
|
||||
if (typeof rowid !== 'number') continue;
|
||||
// ON CONFLICT preserves the rowid, so delete-then-insert replaces the
|
||||
// old FTS row rather than accumulating duplicates for one document.
|
||||
deleteFts.run([rowid]);
|
||||
insertFts.run([rowid, d.title, d.people, d.body]);
|
||||
written++;
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
/** Removes documents by id (a JMAP `destroyed` id, or a stale row). */
|
||||
remove(jmapAccountId: string, contentType: ContentType, ids: readonly string[]): number {
|
||||
if (ids.length === 0) return 0;
|
||||
const findRow = this.db.prepare(
|
||||
'SELECT rowid FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
|
||||
);
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const deleteDoc = this.db.prepare(
|
||||
'DELETE FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
|
||||
);
|
||||
let removed = 0;
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const row = findRow.get([jmapAccountId, contentType, id]);
|
||||
if (typeof row?.rowid === 'number') deleteFts.run([row.rowid]);
|
||||
removed += deleteDoc.run([jmapAccountId, contentType, id]).changes;
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-text search - the retrieval surface an AI feature calls to gather
|
||||
* context. `types` empty/omitted searches everything.
|
||||
*/
|
||||
search(opts: {
|
||||
query: string;
|
||||
types?: readonly ContentType[];
|
||||
limit?: number;
|
||||
snippetTokens?: number;
|
||||
}): SearchHit[] {
|
||||
const match = toFtsMatchQuery(opts.query);
|
||||
if (!match) return [];
|
||||
|
||||
const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200);
|
||||
const tokens = Math.min(Math.max(opts.snippetTokens ?? 24, 4), 64);
|
||||
const types = opts.types && opts.types.length > 0 ? opts.types : null;
|
||||
const typeFilter = types ? ` AND d.content_type IN (${types.map(() => '?').join(',')})` : '';
|
||||
|
||||
// bm25 weights: a hit in the title or in a name/address is a stronger
|
||||
// signal than one in a long body, and for RAG the title is what makes a
|
||||
// retrieved chunk recognisable.
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people,
|
||||
d.occurred_at, d.metadata_json,
|
||||
bm25(doc_fts, 8.0, 4.0, 1.0) AS score,
|
||||
snippet(doc_fts, 2, '[', ']', '…', ${tokens}) AS snip
|
||||
FROM doc_fts
|
||||
JOIN doc d ON d.rowid = doc_fts.rowid
|
||||
WHERE doc_fts MATCH ?${typeFilter}
|
||||
ORDER BY score ASC, d.occurred_at DESC
|
||||
LIMIT ?
|
||||
`)
|
||||
.all([match, ...(types ?? []), limit]);
|
||||
|
||||
return rows.map((r) => ({
|
||||
contentType: String(r.content_type) as ContentType,
|
||||
id: String(r.id),
|
||||
jmapAccountId: String(r.jmap_account_id),
|
||||
title: String(r.title ?? ''),
|
||||
people: String(r.people ?? ''),
|
||||
occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at),
|
||||
metadata: safeParseObject(r.metadata_json),
|
||||
score: typeof r.score === 'number' ? r.score : 0,
|
||||
snippet: String(r.snip ?? ''),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Per-type counts and freshness, for the Settings UI and for debugging. */
|
||||
stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT content_type, COUNT(*) AS n, MAX(occurred_at) AS newest, MAX(indexed_at) AS indexed
|
||||
FROM doc GROUP BY content_type ORDER BY content_type
|
||||
`)
|
||||
.all()
|
||||
.map((r) => ({
|
||||
contentType: String(r.content_type),
|
||||
count: Number(r.n ?? 0),
|
||||
newest: r.newest === null || r.newest === undefined ? null : String(r.newest),
|
||||
indexedAt: typeof r.indexed === 'number' ? r.indexed : null,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Ids already present, so a catch-up pass can skip re-fetching bodies. */
|
||||
existingIds(jmapAccountId: string, contentType: ContentType): Set<string> {
|
||||
const rows = this.db
|
||||
.prepare('SELECT id FROM doc WHERE jmap_account_id = ? AND content_type = ?')
|
||||
.all([jmapAccountId, contentType]);
|
||||
return new Set(rows.map((r) => String(r.id)));
|
||||
}
|
||||
|
||||
/** Drops documents older than the retention floor for a type. */
|
||||
pruneOlderThan(jmapAccountId: string, contentType: ContentType, isoFloor: string): number {
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT rowid FROM doc
|
||||
WHERE jmap_account_id = ? AND content_type = ?
|
||||
AND occurred_at IS NOT NULL AND occurred_at < ?
|
||||
`)
|
||||
.all([jmapAccountId, contentType, isoFloor]);
|
||||
if (rows.length === 0) return 0;
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const deleteDoc = this.db.prepare('DELETE FROM doc WHERE rowid = ?');
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const r of rows) {
|
||||
deleteFts.run([r.rowid]);
|
||||
deleteDoc.run([r.rowid]);
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
}
|
||||
|
||||
function readSchemaVersion(db: SqlcipherDatabase): number | null {
|
||||
try {
|
||||
const row = db.prepare("SELECT v FROM meta WHERE k = 'schema_version'").get();
|
||||
if (!row || row.v === undefined) return null;
|
||||
const n = Number(row.v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
} catch {
|
||||
// `meta` doesn't exist yet - a fresh file.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function safeParseObject(v: unknown): Record<string, unknown> {
|
||||
if (typeof v !== 'string') return {};
|
||||
try {
|
||||
const parsed = JSON.parse(v);
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns arbitrary user text into a safe FTS5 MATCH expression.
|
||||
*
|
||||
* FTS5's query syntax is not SQL, so parameter binding does NOT protect it: a
|
||||
* bare `"` or a stray `*`/`NEAR`/`:` in user input raises
|
||||
* `fts5: syntax error`, which would turn a normal search box into a 500. Every
|
||||
* token is quoted (making it a literal phrase) and a trailing `*` is added to
|
||||
* the last token so typing continues to match as the user types.
|
||||
*
|
||||
* Exported for unit testing - it is the one piece of this file with no
|
||||
* database dependency and the most ways to be wrong.
|
||||
*/
|
||||
export function toFtsMatchQuery(raw: string): string | null {
|
||||
if (typeof raw !== 'string') return null;
|
||||
// Split on anything that isn't a word character or an intra-word mark. Keeps
|
||||
// unicode letters (so "Müller" and "東京" survive) via the u flag.
|
||||
const tokens = raw
|
||||
.normalize('NFC')
|
||||
.split(/[^\p{L}\p{N}_@.'-]+/u)
|
||||
.map((t) => t.replace(/^['-]+|['-]+$/g, ''))
|
||||
.filter((t) => t.length > 0)
|
||||
.slice(0, 24);
|
||||
if (tokens.length === 0) return null;
|
||||
return tokens
|
||||
.map((t, i) => {
|
||||
const quoted = `"${t.replace(/"/g, '""')}"`;
|
||||
// Prefix-match only the final token, and only if it's long enough to not
|
||||
// match half the mailbox.
|
||||
return i === tokens.length - 1 && t.length >= 3 ? `${quoted}*` : quoted;
|
||||
})
|
||||
.join(' AND ');
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
// The read-path fallback. Wraps an `IJMAPClient` so that when a mail read fails
|
||||
// because the network is down, the answer comes from the encrypted offline replica
|
||||
// instead of an empty list.
|
||||
//
|
||||
// WHY THIS SHAPE, AND NOT A CACHE. The replica is consulted ONLY after a read has
|
||||
// genuinely failed at the transport level. That ordering is the whole coherence
|
||||
// story for the design review's H3: the webmail does local delta arithmetic on
|
||||
// mailbox unread counts for mark-read/move/delete, and if the replica sat in FRONT
|
||||
// of the server that arithmetic would operate on replica numbers and need
|
||||
// reconciliation rules. Behind the server, an online session never sees a replica
|
||||
// value at all, and while offline any count drift is bounded and repaired by the
|
||||
// next `Mailbox/changes`.
|
||||
//
|
||||
// WHY IT IS NOT ENOUGH TO LOOK AT THE RESULT. `lib/jmap/client.ts`'s read methods
|
||||
// swallow their own errors and return plausible success: `getEmails()` returns an
|
||||
// empty page, `getEmail()` returns `null`, `getMailboxes()` returns a SYNTHETIC
|
||||
// single Inbox. Falling back on those shapes alone would serve stale replica rows
|
||||
// for a folder the user had genuinely just emptied. So the test is TWO-PART: a
|
||||
// suspicious result AND a `fetch` rejection recorded during that exact call
|
||||
// (`lib/jmap/transport-health.ts`). A 4xx, a 429 or a JMAP method error all mean
|
||||
// the server answered, so none of them triggers a fallback.
|
||||
//
|
||||
// Mutates the instance rather than wrapping it in a Proxy: `JMAPClient` is a large
|
||||
// class whose methods call each other through `this`, and instance patching keeps
|
||||
// `this` identity exactly as it was. Idempotent, so re-wrapping the same client is
|
||||
// harmless.
|
||||
|
||||
import { generateAccountId } from '@/lib/account-utils';
|
||||
import type { IJMAPClient } from '@/lib/jmap/client-interface';
|
||||
import type { Email, Mailbox } from '@/lib/jmap/types';
|
||||
import { transportFailureCount } from '@/lib/jmap/transport-health';
|
||||
import {
|
||||
isReplicaUnavailable, readOfflineList, readOfflineMailboxes, readOfflineMessage,
|
||||
} from '@/lib/offline-replica-client';
|
||||
|
||||
const WRAPPED = Symbol.for('vncmail.offlineFallback.wrapped');
|
||||
|
||||
/**
|
||||
* A `getMailboxes()` result that is really the client's offline placeholder.
|
||||
*
|
||||
* `client.ts` fabricates exactly this on failure: one mailbox, id `INBOX`, role
|
||||
* `inbox`, zero counts. Matching it precisely matters - a real server that happens
|
||||
* to return a single inbox has a real id and real counts.
|
||||
*/
|
||||
function isSyntheticMailboxList(mailboxes: readonly Mailbox[]): boolean {
|
||||
return (
|
||||
mailboxes.length === 1 &&
|
||||
mailboxes[0]?.id === 'INBOX' &&
|
||||
mailboxes[0]?.totalEmails === 0 &&
|
||||
mailboxes[0]?.unreadEmails === 0
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolves this client's cookie slot, so a multi-account shell reads the right replica. */
|
||||
async function slotFor(client: IJMAPClient): Promise<number | undefined> {
|
||||
try {
|
||||
const { useAccountStore } = await import('@/stores/account-store');
|
||||
const id = generateAccountId(client.getUsername(), client.getServerUrl());
|
||||
const accounts = useAccountStore.getState().accounts;
|
||||
const match =
|
||||
accounts.find((a) => a.id === id) ??
|
||||
accounts.find((a) => a.serverIdentifiers?.includes(id));
|
||||
return match?.cookieSlot;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the replica may answer for this call.
|
||||
*
|
||||
* v1 replicates the PRIMARY mail account only, so a read explicitly scoped to a
|
||||
* delegated/shared account must never be answered from it - the replica simply has
|
||||
* no rows, and answering "empty" would be worse than the client's own empty.
|
||||
*/
|
||||
function scopedToPrimary(client: IJMAPClient, accountId?: string): boolean {
|
||||
if (!accountId) return true;
|
||||
try {
|
||||
return accountId === client.getAccountId();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function withOfflineFallback<T extends IJMAPClient>(client: T): T {
|
||||
const flagged = client as unknown as Record<symbol, boolean | undefined>;
|
||||
if (flagged[WRAPPED]) return client;
|
||||
flagged[WRAPPED] = true;
|
||||
|
||||
const target = client as unknown as IJMAPClient;
|
||||
const originalGetEmail = target.getEmail.bind(target);
|
||||
const originalGetEmails = target.getEmails.bind(target);
|
||||
const originalGetMailboxes = target.getMailboxes.bind(target);
|
||||
const originalGetAllMailboxes = target.getAllMailboxes.bind(target);
|
||||
|
||||
target.getEmail = async (emailId: string, accountId?: string): Promise<Email | null> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetEmail(emailId, accountId);
|
||||
if (online) return online;
|
||||
if (isReplicaUnavailable()) return online;
|
||||
// `null` alone is ambiguous: it is also what a genuinely-missing id returns.
|
||||
// Only a transport failure during THIS call earns a fallback.
|
||||
if (transportFailureCount() === before) return online;
|
||||
if (!scopedToPrimary(client, accountId)) return online;
|
||||
|
||||
const offline = await readOfflineMessage(emailId, await slotFor(client));
|
||||
// An envelope with no body would render blank AND leave the viewer's
|
||||
// `isBodyLoading` gate stuck, so it is not an answer - better to keep the
|
||||
// client's `null` and let the UI say the message is unavailable offline.
|
||||
if (!offline?.email || !offline.hasBody) return online;
|
||||
return offline.email;
|
||||
};
|
||||
|
||||
target.getEmails = async (
|
||||
mailboxId?: string,
|
||||
accountId?: string,
|
||||
limit: number = 50,
|
||||
position: number = 0,
|
||||
hasKeyword?: string,
|
||||
pinnedFirst?: boolean,
|
||||
extraFilter?: Record<string, unknown>,
|
||||
): Promise<{ emails: Email[]; hasMore: boolean; total: number }> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetEmails(
|
||||
mailboxId, accountId, limit, position, hasKeyword, pinnedFirst, extraFilter,
|
||||
);
|
||||
if (online.emails.length > 0) return online;
|
||||
if (isReplicaUnavailable()) return online;
|
||||
if (transportFailureCount() === before) return online;
|
||||
if (!scopedToPrimary(client, accountId)) return online;
|
||||
// A keyword or category filter is a server-side query the replica does not
|
||||
// reproduce. Serving an unfiltered page in its place would silently show the
|
||||
// wrong set, which is worse than showing nothing.
|
||||
if (hasKeyword || extraFilter) return online;
|
||||
|
||||
const offline = await readOfflineList(mailboxId ?? null, {
|
||||
limit,
|
||||
offset: position,
|
||||
slot: await slotFor(client),
|
||||
});
|
||||
if (!offline || offline.emails.length === 0) return online;
|
||||
return { emails: offline.emails, hasMore: offline.hasMore, total: offline.total };
|
||||
};
|
||||
|
||||
const mailboxFallback = async (
|
||||
online: Mailbox[],
|
||||
before: number,
|
||||
accountId?: string,
|
||||
): Promise<Mailbox[]> => {
|
||||
// Bail out unless the result is EMPTY or is the exact synthetic placeholder.
|
||||
// Testing `length > 1` here was a real bug found by
|
||||
// `lib/__tests__/offline-fallback-client.test.ts`: a server that legitimately
|
||||
// exposes a single mailbox got its real folder - real id, real counts -
|
||||
// replaced by replica rows the moment any unrelated transport blip was
|
||||
// recorded during the call.
|
||||
if (online.length > 0 && !isSyntheticMailboxList(online)) return online;
|
||||
if (isReplicaUnavailable()) return online;
|
||||
if (transportFailureCount() === before) return online;
|
||||
if (!scopedToPrimary(client, accountId)) return online;
|
||||
const offline = await readOfflineMailboxes(await slotFor(client));
|
||||
if (!offline || offline.length === 0) return online;
|
||||
return offline;
|
||||
};
|
||||
|
||||
target.getMailboxes = async (accountId?: string): Promise<Mailbox[]> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetMailboxes(accountId);
|
||||
return mailboxFallback(online, before, accountId);
|
||||
};
|
||||
|
||||
target.getAllMailboxes = async (): Promise<Mailbox[]> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetAllMailboxes();
|
||||
// `getAllMailboxes` falls back internally to `getMailboxes()`, so an offline
|
||||
// run arrives here as the synthetic single Inbox rather than an empty list.
|
||||
return mailboxFallback(online, before);
|
||||
};
|
||||
|
||||
return client;
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
// Renderer-side client for the offline mail replica.
|
||||
//
|
||||
// The replica is EVENT-DRIVEN, exactly like the search index next to it: the
|
||||
// renderer already holds the live JMAP push connection, so a `StateChange` is what
|
||||
// triggers a sync cycle. There is no polling loop and no background worker.
|
||||
//
|
||||
// One cycle is BOUNDED (see lib/offline-replica/sync.ts's BUDGET), so a first
|
||||
// sync of a large mailbox needs several. `unfinishedWork` is the server saying
|
||||
// "call again", and `chainSync` below does that with a hard cap - the cap matters,
|
||||
// because an "unfinished work" signal that is true for a condition the cycle
|
||||
// cannot change is how the mobile client ended up chaining a new cycle every five
|
||||
// seconds forever.
|
||||
//
|
||||
// Every function here is best-effort and never throws: offline storage failing to
|
||||
// update must never break the mail UI.
|
||||
|
||||
import { apiFetch } from '@/lib/browser-navigation';
|
||||
import { debug } from '@/lib/debug';
|
||||
import type { Email, Mailbox } from '@/lib/jmap/types';
|
||||
import type { StateChange } from '@/lib/jmap/types';
|
||||
|
||||
export interface RetentionPolicy {
|
||||
envelopeDays: number;
|
||||
bodyDays: number;
|
||||
maxBodyMB: number;
|
||||
}
|
||||
|
||||
export interface CycleReport {
|
||||
ok: boolean;
|
||||
unfinishedWork: boolean;
|
||||
bootstrapped: boolean;
|
||||
reconciled: boolean;
|
||||
mailboxesWritten: number;
|
||||
envelopesWritten: number;
|
||||
envelopesDeleted: number;
|
||||
bodiesWritten: number;
|
||||
bodiesEvicted: number;
|
||||
coveragePhase: string;
|
||||
resyncRequired: boolean;
|
||||
warnings: string[];
|
||||
errorClass?: string;
|
||||
error?: string;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export interface ReplicaStats {
|
||||
mailboxes: number;
|
||||
envelopes: number;
|
||||
bodies: number;
|
||||
bodyBytes: number;
|
||||
wantedBodies: number;
|
||||
giveUps: number;
|
||||
newest: string | null;
|
||||
oldest: string | null;
|
||||
fileBytes: number;
|
||||
}
|
||||
|
||||
export interface ReplicaStatus {
|
||||
ok: boolean;
|
||||
policy: RetentionPolicy;
|
||||
limits: Record<keyof RetentionPolicy, { min: number; max: number }>;
|
||||
synced: boolean;
|
||||
stats: ReplicaStats | null;
|
||||
coveragePhase: string;
|
||||
coveredFrom?: string | null;
|
||||
resyncRequired: boolean;
|
||||
lastCycleAt: number | null;
|
||||
lastCycleOk: boolean | null;
|
||||
lastCycleError?: string | null;
|
||||
}
|
||||
|
||||
/** Set once the server says the feature isn't there, so we stop asking. */
|
||||
let knownUnavailable = false;
|
||||
let inFlight: Promise<CycleReport | null> | null = null;
|
||||
|
||||
function slotQuery(slot?: number, extra?: string): string {
|
||||
const params = new URLSearchParams();
|
||||
if (typeof slot === 'number') params.set('slot', String(slot));
|
||||
const base = params.toString();
|
||||
if (extra && base) return `?${base}&${extra}`;
|
||||
if (extra) return `?${extra}`;
|
||||
return base ? `?${base}` : '';
|
||||
}
|
||||
|
||||
/** True when the replica is known to be absent (not the desktop shell, or gated off). */
|
||||
export function isReplicaUnavailable(): boolean {
|
||||
return knownUnavailable;
|
||||
}
|
||||
|
||||
export function resetReplicaAvailability(): void {
|
||||
knownUnavailable = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs ONE cycle. Single-flighted on the renderer as well as the server, so a
|
||||
* burst of deliveries coalesces instead of queueing N overlapping requests that
|
||||
* the server would then serialise anyway.
|
||||
*/
|
||||
export async function syncOnce(
|
||||
opts: { slot?: number; policy?: RetentionPolicy; forceResync?: boolean } = {},
|
||||
): Promise<CycleReport | null> {
|
||||
if (knownUnavailable) return null;
|
||||
if (inFlight) return inFlight;
|
||||
|
||||
const run = (async (): Promise<CycleReport | null> => {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/sync${slotQuery(opts.slot)}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ policy: opts.policy, forceResync: opts.forceResync === true }),
|
||||
});
|
||||
// 404 = not the desktop shell. Permanent for this page load; stop asking so
|
||||
// a busy mailbox doesn't post per delivery.
|
||||
if (response.status === 404) { knownUnavailable = true; return null; }
|
||||
if (response.status === 503) {
|
||||
const body = await response.json().catch(() => ({}));
|
||||
// A transport-class 503 means the BACKEND is unreachable, which is normal
|
||||
// and temporary - it must not latch the feature off for the session. Only
|
||||
// a missing binding / key channel does that.
|
||||
const code = typeof body?.code === 'string' ? body.code : '';
|
||||
if (code === 'no-binding' || code === 'no-key-channel' || code === 'unavailable') {
|
||||
knownUnavailable = true;
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) return null;
|
||||
const body = await response.json();
|
||||
debug.log('push', '[replica] cycle', body?.report);
|
||||
return (body?.report ?? null) as CycleReport | null;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
inFlight = null;
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
/** Hard cap on chained cycles per trigger. */
|
||||
export const MAX_CHAINED_CYCLES = 12;
|
||||
|
||||
/**
|
||||
* Runs cycles while the server reports unfinished work.
|
||||
*
|
||||
* The cap is the whole point. `unfinishedWork` is a hint, and a hint that stays
|
||||
* true for something the cycle cannot resolve turns into an endless chain - which
|
||||
* is exactly what happened on the mobile client when a body-queue counter reported
|
||||
* attempted rather than inserted rows. The server-side fixes make that
|
||||
* self-terminating; this cap means even a future regression costs a bounded number
|
||||
* of requests rather than an infinite loop.
|
||||
*/
|
||||
export async function chainSync(
|
||||
opts: { slot?: number; max?: number; onReport?: (report: CycleReport) => void } = {},
|
||||
): Promise<CycleReport | null> {
|
||||
const max = Math.min(opts.max ?? MAX_CHAINED_CYCLES, MAX_CHAINED_CYCLES);
|
||||
let last: CycleReport | null = null;
|
||||
for (let i = 0; i < max; i++) {
|
||||
const report = await syncOnce({ slot: opts.slot });
|
||||
if (!report) return last;
|
||||
last = report;
|
||||
opts.onReport?.(report);
|
||||
if (!report.ok || !report.unfinishedWork) return report;
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
/** The push-driven entry point. Fire-and-forget: the mail UI must not wait on it. */
|
||||
export function syncOnStateChange(change: StateChange, opts: { slot?: number } = {}): void {
|
||||
if (knownUnavailable) return;
|
||||
// Only mail-shaped changes are worth a cycle. A `Mailbox` state change alone is
|
||||
// usually just an unread-count move, but the replica DOES hold those counts, so
|
||||
// unlike the search index it is worth reacting to.
|
||||
const relevant = Object.values(change.changed ?? {}).some(
|
||||
(perAccount) => perAccount && (perAccount.Email || perAccount.Mailbox),
|
||||
);
|
||||
if (!relevant) return;
|
||||
void syncOnce({ slot: opts.slot });
|
||||
}
|
||||
|
||||
export async function fetchReplicaStatus(slot?: number): Promise<ReplicaStatus | null> {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`);
|
||||
if (!response.ok) return null;
|
||||
return (await response.json()) as ReplicaStatus;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateRetentionPolicy(
|
||||
policy: RetentionPolicy,
|
||||
slot?: number,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(policy),
|
||||
});
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function purgeReplica(slot?: number): Promise<boolean> {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`, { method: 'DELETE' });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── reads ───────────────────────────────────────────────────────────────────
|
||||
|
||||
interface ReadEnvelope<T> {
|
||||
ok?: boolean;
|
||||
available?: boolean;
|
||||
error?: string;
|
||||
data?: T;
|
||||
}
|
||||
|
||||
async function read<T>(query: string): Promise<(T & { available: boolean }) | null> {
|
||||
if (knownUnavailable) return null;
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/mail${query}`);
|
||||
if (response.status === 404) { knownUnavailable = true; return null; }
|
||||
if (!response.ok) return null;
|
||||
const body = (await response.json()) as ReadEnvelope<unknown> & Record<string, unknown>;
|
||||
if (body?.available !== true) return null;
|
||||
return body as unknown as T & { available: boolean };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readOfflineMailboxes(slot?: number): Promise<Mailbox[] | null> {
|
||||
const body = await read<{ mailboxes: Mailbox[] }>(slotQuery(slot, 'kind=mailboxes'));
|
||||
return body?.mailboxes ?? null;
|
||||
}
|
||||
|
||||
export async function readOfflineList(
|
||||
mailboxId: string | null,
|
||||
opts: { limit?: number; offset?: number; slot?: number } = {},
|
||||
): Promise<{ emails: Email[]; total: number; hasMore: boolean } | null> {
|
||||
const params = new URLSearchParams({ kind: 'list' });
|
||||
if (mailboxId !== null) params.set('mailboxId', mailboxId);
|
||||
if (opts.limit !== undefined) params.set('limit', String(opts.limit));
|
||||
if (opts.offset !== undefined) params.set('offset', String(opts.offset));
|
||||
const body = await read<{ emails: Email[]; total: number; hasMore: boolean }>(
|
||||
slotQuery(opts.slot, params.toString()),
|
||||
);
|
||||
if (!body) return null;
|
||||
return { emails: body.emails ?? [], total: body.total ?? 0, hasMore: body.hasMore === true };
|
||||
}
|
||||
|
||||
export async function readOfflineMessage(
|
||||
id: string,
|
||||
slot?: number,
|
||||
): Promise<{ email: Email | null; hasBody: boolean } | null> {
|
||||
const params = new URLSearchParams({ kind: 'message', id });
|
||||
const body = await read<{ email: Email | null; hasBody: boolean }>(
|
||||
slotQuery(slot, params.toString()),
|
||||
);
|
||||
if (!body) return null;
|
||||
return { email: body.email ?? null, hasBody: body.hasBody === true };
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
advanceOneMs, madeForwardProgress, normalisePage, pageIsEmpty, planEmailFetches,
|
||||
planMailboxFetches, updatedPropertiesAreCountsOnly, type ChangesPage,
|
||||
} from '../apply';
|
||||
import { asChangesState } from '../states';
|
||||
|
||||
function page(partial: Partial<ChangesPage>): ChangesPage {
|
||||
return {
|
||||
oldState: asChangesState('old'),
|
||||
newState: asChangesState('new'),
|
||||
hasMoreChanges: false,
|
||||
created: [],
|
||||
updated: [],
|
||||
destroyed: [],
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
describe('normalisePage', () => {
|
||||
it('lets a destroyed id win outright over created and updated', () => {
|
||||
// Fetching an id that is also destroyed spends a request to get `notFound`.
|
||||
const out = normalisePage(page({ created: ['a', 'b'], updated: ['a'], destroyed: ['a'] }));
|
||||
expect(out.created).toEqual(['b']);
|
||||
expect(out.updated).toEqual([]);
|
||||
expect(out.destroyed).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('treats an id in both created and updated as a create', () => {
|
||||
// The create path fetches the full envelope tier, which already contains the
|
||||
// updated values - so an extra 3-property fetch would be pure waste.
|
||||
const out = normalisePage(page({ created: ['a'], updated: ['a'] }));
|
||||
expect(out.created).toEqual(['a']);
|
||||
expect(out.updated).toEqual([]);
|
||||
});
|
||||
|
||||
it('deduplicates within each bucket', () => {
|
||||
const out = normalisePage(page({ created: ['a', 'a'], destroyed: ['b', 'b'] }));
|
||||
expect(out.created).toEqual(['a']);
|
||||
expect(out.destroyed).toEqual(['b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pageIsEmpty', () => {
|
||||
it('is true only when nothing changed', () => {
|
||||
// An empty page STILL has to advance the cursor: skipping it re-requests the
|
||||
// same position forever.
|
||||
expect(pageIsEmpty(page({}))).toBe(true);
|
||||
expect(pageIsEmpty(page({ updated: ['a'] }))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('planEmailFetches', () => {
|
||||
it('drops an updated id we do not hold locally, BEFORE any fetch is issued', () => {
|
||||
// The absent case is an unconditional no-op. Fetching it would need a
|
||||
// `receivedAt` the 3-property response cannot supply and the schema's
|
||||
// NOT NULL would reject. Coverage enumerates CURRENT state, so it will pick
|
||||
// the record up with the updated values anyway.
|
||||
const plan = planEmailFetches(page({ updated: ['have', 'missing'] }), new Set(['have']));
|
||||
expect(plan.updateIds).toEqual(['have']);
|
||||
});
|
||||
|
||||
it('keeps creates unconditional - presence is irrelevant for a create', () => {
|
||||
const plan = planEmailFetches(page({ created: ['new'] }), new Set());
|
||||
expect(plan.createIds).toEqual(['new']);
|
||||
});
|
||||
|
||||
it('never routes an id into both the create and the update fetch', () => {
|
||||
const plan = planEmailFetches(page({ created: ['a'], updated: ['a'] }), new Set(['a']));
|
||||
expect(plan.createIds).toEqual(['a']);
|
||||
expect(plan.updateIds).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updatedPropertiesAreCountsOnly', () => {
|
||||
it('is true for the four counters and for an empty list', () => {
|
||||
expect(updatedPropertiesAreCountsOnly(['unreadEmails'])).toBe(true);
|
||||
expect(updatedPropertiesAreCountsOnly(['totalEmails', 'unreadThreads'])).toBe(true);
|
||||
// "nothing but the state token moved" is counts-only vacuously.
|
||||
expect(updatedPropertiesAreCountsOnly([])).toBe(true);
|
||||
});
|
||||
|
||||
it('is false when the server will not say what changed', () => {
|
||||
// `null` means "assume everything", so the whole object must be re-fetched.
|
||||
expect(updatedPropertiesAreCountsOnly(null)).toBe(false);
|
||||
expect(updatedPropertiesAreCountsOnly(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it('is false as soon as one non-count property is present', () => {
|
||||
expect(updatedPropertiesAreCountsOnly(['unreadEmails', 'name'])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('planMailboxFetches', () => {
|
||||
it('routes updates to the cheap four-integer patch when only counts moved', () => {
|
||||
const plan = planMailboxFetches(
|
||||
page({ created: ['new'], updated: ['old'], updatedProperties: ['unreadEmails'] }),
|
||||
);
|
||||
expect(plan.fullIds).toEqual(['new']);
|
||||
expect(plan.countOnlyIds).toEqual(['old']);
|
||||
});
|
||||
|
||||
it('re-fetches the whole object when updatedProperties is null', () => {
|
||||
const plan = planMailboxFetches(page({ updated: ['old'], updatedProperties: null }));
|
||||
expect(plan.fullIds).toEqual(['old']);
|
||||
expect(plan.countOnlyIds).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('keyset progress', () => {
|
||||
it('requires STRICTLY greater, because `after` is spec-inclusive', () => {
|
||||
// RFC 8621 s4.4.1: receivedAt "must be the same or after this date-time to
|
||||
// match". So every page re-returns the boundary message, and equality is NOT
|
||||
// progress - treating it as progress would loop on that millisecond forever.
|
||||
expect(madeForwardProgress('2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z')).toBe(false);
|
||||
expect(madeForwardProgress('2026-01-01T00:00:00.001Z', '2026-01-01T00:00:00.000Z')).toBe(true);
|
||||
expect(madeForwardProgress(null, '2026-01-01T00:00:00.000Z')).toBe(false);
|
||||
expect(madeForwardProgress('2026-01-01T00:00:00.000Z', null)).toBe(true);
|
||||
});
|
||||
|
||||
it('advances exactly one millisecond in the last-resort rung', () => {
|
||||
expect(advanceOneMs('2026-01-01T00:00:00.000Z')).toBe('2026-01-01T00:00:00.001Z');
|
||||
// A malformed value must not become NaN and poison the cursor.
|
||||
expect(advanceOneMs('not-a-date')).toBe('not-a-date');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
backoffDelayMs, classify, escalationApplies, movesCursor, nextRung, rungValue,
|
||||
type ErrorClass,
|
||||
} from '../errors';
|
||||
|
||||
const ALL: ErrorClass[] = [
|
||||
'Transport', 'RateLimit', 'ServerTransient', 'RequestLimit', 'Auth', 'Fatal', 'StateInvalid',
|
||||
];
|
||||
|
||||
describe('exactly one class moves a cursor', () => {
|
||||
it('is StateInvalid, and nothing else', () => {
|
||||
// This is the single load-bearing property of the taxonomy. Every other class
|
||||
// leaves the cursor exactly where it was, which is what makes "a failure never
|
||||
// causes silent data loss" structural rather than aspirational.
|
||||
expect(ALL.filter(movesCursor)).toEqual(['StateInvalid']);
|
||||
});
|
||||
|
||||
it('escalates to a rebuild only for size/availability problems', () => {
|
||||
// Escalating on RateLimit would answer a rate-limited server with far MORE
|
||||
// requests. On Auth, a 401 would trigger a rebuild. On Transport, a flaky
|
||||
// tunnel would. Fatal is our own bug and a rebuild will not fix it.
|
||||
expect(ALL.filter(escalationApplies).sort()).toEqual(['RequestLimit', 'ServerTransient']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('classify', () => {
|
||||
it('reads HTTP status before anything else', () => {
|
||||
expect(classify({ httpStatus: 401 })).toBe('Auth');
|
||||
expect(classify({ httpStatus: 403 })).toBe('Auth');
|
||||
expect(classify({ httpStatus: 429 })).toBe('RateLimit');
|
||||
expect(classify({ httpStatus: 413 })).toBe('RequestLimit');
|
||||
expect(classify({ httpStatus: 503 })).toBe('ServerTransient');
|
||||
});
|
||||
|
||||
it('classifies cannotCalculateChanges as the one cursor-moving class', () => {
|
||||
expect(classify({ jmapErrorType: 'cannotCalculateChanges' })).toBe('StateInvalid');
|
||||
});
|
||||
|
||||
it('defaults an UNRECOGNISED method error to ServerTransient', () => {
|
||||
// Guessing transient costs a retry; guessing state-invalid costs a full
|
||||
// resync; guessing fatal stalls the account. The cheapest wrong answer wins.
|
||||
expect(classify({ jmapErrorType: 'somethingNobodyHasHeardOf' })).toBe('ServerTransient');
|
||||
});
|
||||
|
||||
it('does not let a method error description masquerade as a transport failure', () => {
|
||||
// Structure before strings: a method error's prose can legitimately contain
|
||||
// "timeout" or "socket", and reading that as Transport would leave a genuine
|
||||
// server-side problem being retried as though the network were down.
|
||||
expect(classify({ jmapErrorType: 'invalidArguments', message: 'socket timeout' })).toBe('Fatal');
|
||||
});
|
||||
|
||||
it('classifies a real fetch rejection as Transport', () => {
|
||||
// "Offline is not an error": the cursor stands still and the work is retried.
|
||||
for (const message of [
|
||||
'fetch failed', 'connect ECONNREFUSED 127.0.0.1:1', 'getaddrinfo ENOTFOUND nope',
|
||||
'socket hang up', 'The operation timed out',
|
||||
]) {
|
||||
expect(classify({ message }), message).toBe('Transport');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('the maxChanges ladder is monotonically non-increasing for EVERY server value', () => {
|
||||
it('never proposes a retry larger than the attempt that just failed', () => {
|
||||
// Two historical bugs live here. An unbounded middle rung produced a retry
|
||||
// STRICTLY LARGER than the failing attempt, actively worsening a
|
||||
// "response too large" error. Clamping only rung 0 then reintroduced it in a
|
||||
// narrower form: maxObjectsInGet=100 gave rung0=100 and rung1=250.
|
||||
const serverValues = [
|
||||
undefined, 1, 5, 10, 20, 25, 26, 49, 50, 51, 99, 100, 249, 250, 251, 499, 500, 501, 5000,
|
||||
];
|
||||
for (const value of serverValues) {
|
||||
const rungs = ([0, 1, 2, 3] as const).map((r) => rungValue(r, value));
|
||||
for (let i = 1; i < rungs.length; i++) {
|
||||
expect(
|
||||
rungs[i],
|
||||
`maxObjectsInGet=${value} rung ${i} (${rungs[i]}) must not exceed rung ${i - 1} (${rungs[i - 1]})`,
|
||||
).toBeLessThanOrEqual(rungs[i - 1]);
|
||||
}
|
||||
// And never zero, or the request asks for nothing and never progresses.
|
||||
for (const r of rungs) expect(r).toBeGreaterThanOrEqual(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('clamps rung 0 to what the server allows', () => {
|
||||
expect(rungValue(0, 100)).toBe(100);
|
||||
expect(rungValue(0, 5000)).toBe(500);
|
||||
expect(rungValue(0, undefined)).toBe(500);
|
||||
});
|
||||
|
||||
it('saturates rather than running off the end of the ladder', () => {
|
||||
expect(nextRung(0)).toBe(1);
|
||||
expect(nextRung(3)).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('backoff', () => {
|
||||
it('is full-jitter and bounded by the cap', () => {
|
||||
for (let attempt = 0; attempt < 12; attempt++) {
|
||||
const delay = backoffDelayMs(attempt, { baseMs: 1000, capMs: 60_000 });
|
||||
expect(delay).toBeGreaterThanOrEqual(0);
|
||||
expect(delay).toBeLessThanOrEqual(60_000);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
// The clock-jump guard, and the wipe it caused on the mobile client.
|
||||
//
|
||||
// The bug being regressed here is not hypothetical: its reproduction on the mobile
|
||||
// side returned 0 envelopes from a 4-envelope store. The guard DETECTED the jump,
|
||||
// held the old floor for one cycle - and persisted the JUMPED floor. The next
|
||||
// chained cycle seconds later computed a floor within seconds of the persisted
|
||||
// one, so the guard passed, the movement was classified as a NARROW, and every
|
||||
// envelope below a floor a year in the future was evicted. Unrecoverable, because
|
||||
// `coveredFrom` then claims the range complete and `/changes` cannot re-deliver
|
||||
// pre-existing mail.
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
adjustForWindow, CLOCK_JUMP_GUARD_MS, computeFloors, floorMovement,
|
||||
guardFloorAgainstClockJump,
|
||||
} from '../retention';
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
const T0 = Date.parse('2026-08-05T12:00:00.000Z');
|
||||
|
||||
function iso(t: number): string {
|
||||
return new Date(t).toISOString();
|
||||
}
|
||||
|
||||
describe('computeFloors', () => {
|
||||
it('never lets the body window be wider than the envelope window', () => {
|
||||
// A body with no envelope is an orphan by construction, and the whole point of
|
||||
// two tiers is envelopes being a superset of bodies.
|
||||
const floors = computeFloors({ envelopeDays: 30, bodyDays: 365, maxBodyMB: 100 }, T0);
|
||||
expect(floors.bodyFrom).toBe(floors.envelopeFrom);
|
||||
});
|
||||
|
||||
it('turns the MB cap into bytes', () => {
|
||||
expect(computeFloors({ envelopeDays: 1, bodyDays: 1, maxBodyMB: 2 }, T0).maxBodyBytes)
|
||||
.toBe(2 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe('guardFloorAgainstClockJump', () => {
|
||||
it('adopts the computed floor when there is no history to compare against', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0), undefined);
|
||||
expect(g.suppressed).toBe(false);
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
expect(g.envelopeFrom).toBe(iso(T0));
|
||||
});
|
||||
|
||||
it('adopts an ordinary drift - a DST shift must not trip it', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0 + 60 * 60 * 1000), iso(T0));
|
||||
expect(g.suppressed).toBe(false);
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
});
|
||||
|
||||
it('suppresses a jump larger than the guard and refuses to authorise deletion', () => {
|
||||
const jumped = iso(T0 + 365 * DAY);
|
||||
const g = guardFloorAgainstClockJump(jumped, iso(T0));
|
||||
expect(g.suppressed).toBe(true);
|
||||
expect(g.envelopeFrom).toBe(iso(T0));
|
||||
// Suppressing the FLOOR is not the same as suppressing the DELETIONS the
|
||||
// floor authorises. Both the retention eviction and the reconcile sweep read
|
||||
// this bit.
|
||||
expect(g.evictionAllowed).toBe(false);
|
||||
expect(g.warning).toBeTruthy();
|
||||
});
|
||||
|
||||
it('THE H2 REGRESSION: persists the floor it USED, not the one it rejected', () => {
|
||||
// This one assertion is the whole fix. Persisting the computed value here is
|
||||
// what legitimised the anomaly on the very next cycle.
|
||||
const jumped = iso(T0 + 365 * DAY);
|
||||
const g = guardFloorAgainstClockJump(jumped, iso(T0));
|
||||
expect(g.nextLastWindowFloor).toBe(iso(T0));
|
||||
expect(g.nextLastWindowFloor).not.toBe(jumped);
|
||||
});
|
||||
|
||||
it('THE H2 REGRESSION: stays suppressed across MANY chained cycles', () => {
|
||||
// The original bug only showed on the SECOND cycle, so a single-cycle test
|
||||
// passes against the broken code. Chaining is what reproduces it.
|
||||
const stored = iso(T0);
|
||||
let lastWindowFloor: string | undefined = stored;
|
||||
for (let cycle = 0; cycle < 20; cycle++) {
|
||||
// The clock is a year ahead and creeping forward a few seconds per cycle,
|
||||
// exactly as a chained sync would observe it.
|
||||
const computed = iso(T0 + 365 * DAY + cycle * 5_000);
|
||||
const g = guardFloorAgainstClockJump(computed, lastWindowFloor);
|
||||
expect(g.suppressed, `cycle ${cycle} must stay suppressed`).toBe(true);
|
||||
expect(g.evictionAllowed, `cycle ${cycle} must not authorise deletion`).toBe(false);
|
||||
expect(g.envelopeFrom, `cycle ${cycle} must keep the original floor`).toBe(stored);
|
||||
lastWindowFloor = g.nextLastWindowFloor;
|
||||
}
|
||||
// And after 20 cycles the remembered floor is still the trustworthy one, so
|
||||
// no later cycle can classify it as a narrow and evict everything.
|
||||
expect(lastWindowFloor).toBe(stored);
|
||||
expect(floorMovement(lastWindowFloor, stored)).toBe('unchanged');
|
||||
expect(adjustForWindow(floorMovement(lastWindowFloor, stored), stored).evictBelow)
|
||||
.toBeUndefined();
|
||||
});
|
||||
|
||||
it('suppresses a BACKWARD jump too', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0 - 365 * DAY), iso(T0));
|
||||
expect(g.suppressed).toBe(true);
|
||||
expect(g.evictionAllowed).toBe(false);
|
||||
});
|
||||
|
||||
it('treats an explicit retention change as INTENT and applies it, eviction included', () => {
|
||||
// The computed floor moves for two independent reasons - the clock changing
|
||||
// and the SETTING changing - and guarding a setting change is wrong. Without
|
||||
// this discriminator a Settings edit sits unapplied until something unrelated
|
||||
// moves the floor again.
|
||||
const widened = iso(T0 - 365 * DAY);
|
||||
const g = guardFloorAgainstClockJump(widened, iso(T0), { policyChanged: true });
|
||||
expect(g.suppressed).toBe(false);
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
expect(g.envelopeFrom).toBe(widened);
|
||||
expect(g.nextLastWindowFloor).toBe(widened);
|
||||
});
|
||||
|
||||
it('a genuine user NARROW still evicts', () => {
|
||||
// The guard must not become a reason nothing is ever deleted.
|
||||
const narrowed = iso(T0 + 20 * 60 * 60 * 1000);
|
||||
const g = guardFloorAgainstClockJump(narrowed, iso(T0));
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
expect(floorMovement(iso(T0), g.envelopeFrom)).toBe('narrowed');
|
||||
expect(adjustForWindow('narrowed', g.envelopeFrom).evictBelow).toBe(narrowed);
|
||||
});
|
||||
|
||||
it('tolerates an unparseable stored floor without wedging', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0), 'garbage');
|
||||
// Date.parse('garbage') is NaN, so the delta is not finite: adopt rather than
|
||||
// suppress forever on a corrupt value.
|
||||
expect(g.suppressed).toBe(false);
|
||||
});
|
||||
|
||||
it('uses a threshold above a day so a leap second or NTP nudge is invisible', () => {
|
||||
expect(CLOCK_JUMP_GUARD_MS).toBeGreaterThan(DAY);
|
||||
});
|
||||
});
|
||||
|
||||
describe('floorMovement / adjustForWindow', () => {
|
||||
it('a LATER floor keeps less mail and means evict', () => {
|
||||
expect(floorMovement(iso(T0), iso(T0 + DAY))).toBe('narrowed');
|
||||
expect(adjustForWindow('narrowed', iso(T0 + DAY))).toEqual({ evictBelow: iso(T0 + DAY) });
|
||||
});
|
||||
|
||||
it('an EARLIER floor means re-scan, NOT a resync', () => {
|
||||
// A widen moves the target back and re-enters coverage scanning. The cursors
|
||||
// are untouched - a widen is not a reason to rebuild.
|
||||
expect(floorMovement(iso(T0), iso(T0 - DAY))).toBe('widened');
|
||||
expect(adjustForWindow('widened', iso(T0 - DAY))).toEqual({ rescanFrom: iso(T0 - DAY) });
|
||||
});
|
||||
|
||||
it('does nothing without a previous floor', () => {
|
||||
expect(floorMovement(undefined, iso(T0))).toBe('unchanged');
|
||||
expect(adjustForWindow('unchanged', iso(T0))).toEqual({});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
asChangesState, asSnapshotState, coveragePhaseForCommitment, mintEnumerationCommitment,
|
||||
} from '../states';
|
||||
|
||||
describe('state token certification', () => {
|
||||
it('rejects everything a parsed JSON body could hand over that is not a token', () => {
|
||||
// The brand certifies PROVENANCE; this check certifies SHAPE. Without it a
|
||||
// `null` or a number could be laundered into something the engine treats as a
|
||||
// cursor forever.
|
||||
for (const bad of [null, undefined, 0, 1, '', {}, [], true]) {
|
||||
expect(() => asChangesState(bad)).toThrow(TypeError);
|
||||
expect(() => asSnapshotState(bad)).toThrow(TypeError);
|
||||
}
|
||||
});
|
||||
|
||||
it('accepts a non-empty string', () => {
|
||||
expect(asChangesState('s1')).toBe('s1');
|
||||
expect(asSnapshotState('s1')).toBe('s1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('EnumerationCommitment', () => {
|
||||
it('is constructible - the symbol tag must be a real runtime Symbol', () => {
|
||||
// `declare const tag: unique symbol` is type-level only and emits no runtime
|
||||
// value, so using it as a computed key throws ReferenceError the first time
|
||||
// the mint runs. That mistake is in the superseded design document; this test
|
||||
// is what catches it.
|
||||
const commitment = mintEnumerationCommitment({
|
||||
jmapAccountId: 'a',
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: '2026-01-01T00:00:00.000Z',
|
||||
sweepFloor: '2026-01-01T00:00:00.000Z',
|
||||
kind: 'bootstrap',
|
||||
});
|
||||
expect(commitment.snapshot).toBe('snap');
|
||||
expect(commitment.kind).toBe('bootstrap');
|
||||
});
|
||||
|
||||
it('maps its kind onto the coverage phase', () => {
|
||||
const base = {
|
||||
jmapAccountId: 'a',
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: 'x',
|
||||
sweepFloor: 'x',
|
||||
} as const;
|
||||
expect(coveragePhaseForCommitment(mintEnumerationCommitment({ ...base, kind: 'bootstrap' })))
|
||||
.toBe('scanning');
|
||||
expect(coveragePhaseForCommitment(mintEnumerationCommitment({ ...base, kind: 'reconcile' })))
|
||||
.toBe('reconciling');
|
||||
});
|
||||
|
||||
it('does not export its tag, so no object literal elsewhere can forge the type', () => {
|
||||
const source = fs.readFileSync(path.join(__dirname, '..', 'states.ts'), 'utf8');
|
||||
expect(source).toContain("const enumerationCommitmentTag = Symbol('EnumerationCommitment')");
|
||||
expect(source).not.toMatch(/export\s+(const|let)\s+enumerationCommitmentTag/);
|
||||
// And it must be a real Symbol() call, not the type-only declaration form.
|
||||
expect(source).not.toMatch(/declare\s+const\s+enumerationCommitmentTag/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cursor provenance is greppable, not just documented', () => {
|
||||
const replicaDir = path.join(__dirname, '..');
|
||||
|
||||
function sourceFiles(): string[] {
|
||||
return fs
|
||||
.readdirSync(replicaDir)
|
||||
.filter((f) => f.endsWith('.ts'))
|
||||
.map((f) => path.join(replicaDir, f));
|
||||
}
|
||||
|
||||
it('mints branded states ONLY in jmap.ts (the response parser)', () => {
|
||||
// This is the rule the whole brand exists to enforce. The mobile client's
|
||||
// defect D4 was a snapshot state adopted as a /changes cursor after a
|
||||
// transient 503; a cast anywhere outside the parser is how that comes back.
|
||||
for (const file of sourceFiles()) {
|
||||
const base = path.basename(file);
|
||||
if (base === 'states.ts' || base === 'jmap.ts') continue;
|
||||
const source = fs.readFileSync(file, 'utf8');
|
||||
expect(source, `${base} must not mint a ChangesState`).not.toMatch(/asChangesState\s*\(/);
|
||||
expect(source, `${base} must not mint a SnapshotState`).not.toMatch(/asSnapshotState\s*\(/);
|
||||
expect(source, `${base} must not cast to a branded state`).not.toMatch(
|
||||
/as\s+(ChangesState|SnapshotState)\b/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('mints an EnumerationCommitment ONLY where an enumeration is actually started', () => {
|
||||
// A commitment is a promise to enumerate. Minting one anywhere that does not
|
||||
// then enumerate makes the seed path's teeth meaningless.
|
||||
const callers = sourceFiles().filter((file) => {
|
||||
if (path.basename(file) === 'states.ts') return false;
|
||||
return /mintEnumerationCommitment\s*\(/.test(fs.readFileSync(file, 'utf8'));
|
||||
});
|
||||
expect(callers.map((f) => path.basename(f))).toEqual(['sync.ts']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,502 @@
|
||||
// Store-level invariants, against a REAL SQLCipher file.
|
||||
//
|
||||
// Skipped wholesale when the optional native binding is not installed (that is a
|
||||
// normal state on a platform with no prebuild - see lib/mail-index/binding.ts), so
|
||||
// this file must never be the only proof of anything.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
||||
import { indexDbPath } from '@/lib/mail-index/paths';
|
||||
import { clampPolicy, DEFAULT_POLICY, ReplicaStore } from '../store';
|
||||
import { reconcileStamp } from '../sync';
|
||||
import { asChangesState, asSnapshotState, mintEnumerationCommitment } from '../states';
|
||||
import type { EnvelopeRow } from '../types';
|
||||
|
||||
const ACCOUNT = 'alice@example.org';
|
||||
const JMAP = 'jmap-account-1';
|
||||
|
||||
function envelope(id: string, receivedAt: string, extra: Partial<EnvelopeRow> = {}): EnvelopeRow {
|
||||
return {
|
||||
jmapAccountId: JMAP,
|
||||
id,
|
||||
threadId: `t-${id}`,
|
||||
receivedAt,
|
||||
size: 1000,
|
||||
subject: `subject ${id}`,
|
||||
preview: `preview ${id}`,
|
||||
fromJson: JSON.stringify([{ email: 'sender@example.org' }]),
|
||||
toJson: null,
|
||||
ccJson: null,
|
||||
blobId: `blob-${id}`,
|
||||
hasAttachment: false,
|
||||
keywordsJson: '{}',
|
||||
mailboxIds: ['inbox'],
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
describe.skipIf(!isSqlcipherAvailable())('ReplicaStore', () => {
|
||||
let storeDir: string;
|
||||
let key: Buffer;
|
||||
let store: ReplicaStore;
|
||||
|
||||
beforeEach(() => {
|
||||
storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-test-'));
|
||||
key = randomBytes(32);
|
||||
store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
store.close();
|
||||
fs.rmSync(storeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes into the SAME file as the search index, and it is really encrypted', () => {
|
||||
// One encryption boundary, one key, one purge. And `PRAGMA key` is a silent
|
||||
// no-op on a non-SQLCipher binding, so the header check is the only thing that
|
||||
// catches a store that "works" while sitting on disk in cleartext.
|
||||
expect(store.dbPath).toBe(indexDbPath(storeDir, ACCOUNT));
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.close();
|
||||
const header = fs.readFileSync(store.dbPath).subarray(0, 15).toString('latin1');
|
||||
expect(header).not.toBe('SQLite format 3');
|
||||
const raw = Buffer.concat(
|
||||
['', '-wal', '-shm']
|
||||
.map((s) => `${store.dbPath}${s}`)
|
||||
.filter((f) => fs.existsSync(f))
|
||||
.map((f) => fs.readFileSync(f)),
|
||||
);
|
||||
expect(raw.includes('subject e1')).toBe(false);
|
||||
// Re-open so afterEach's close() is harmless.
|
||||
store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
|
||||
});
|
||||
|
||||
describe('cursor provenance at the storage layer', () => {
|
||||
it('refuses to create a cursor from nowhere', () => {
|
||||
// A cursor is born from seedCursor and nowhere else. Creating one in
|
||||
// advanceCursor would be a silent cursor-from-nowhere - exactly what the
|
||||
// branded types exist to make impossible.
|
||||
expect(() => store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s1')))
|
||||
.toThrow(/seed it first/);
|
||||
});
|
||||
|
||||
it('writes the cursor AND the coverage row it justifies in one transaction', () => {
|
||||
store.transaction(() => {
|
||||
store.seedCursor(
|
||||
{ jmapAccountId: JMAP, type: 'Email' },
|
||||
mintEnumerationCommitment({
|
||||
jmapAccountId: JMAP,
|
||||
snapshot: asSnapshotState('snap-1'),
|
||||
targetFrom: '2026-01-01T00:00:00.000Z',
|
||||
sweepFloor: '2026-01-01T00:00:00.000Z',
|
||||
kind: 'bootstrap',
|
||||
}),
|
||||
1000,
|
||||
);
|
||||
});
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('snap-1');
|
||||
const coverage = store.getCoverage(JMAP);
|
||||
expect(coverage?.phase).toBe('scanning');
|
||||
expect(coverage?.sweepFloor).toBe('2026-01-01T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('rolls back a seed whose commitment is for the wrong account', () => {
|
||||
expect(() =>
|
||||
store.transaction(() => {
|
||||
store.seedCursor(
|
||||
{ jmapAccountId: JMAP, type: 'Email' },
|
||||
mintEnumerationCommitment({
|
||||
jmapAccountId: 'someone-else',
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: 'x', sweepFloor: 'x', kind: 'bootstrap',
|
||||
}),
|
||||
1000,
|
||||
);
|
||||
}),
|
||||
).toThrow(/different JMAP account/);
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull();
|
||||
});
|
||||
|
||||
it('advances a seeded cursor and keeps counters field-level', () => {
|
||||
seed(store);
|
||||
store.transaction(() => {
|
||||
store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s2'));
|
||||
store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { consecutiveFailures: 3 });
|
||||
});
|
||||
const cursor = store.getCursor({ jmapAccountId: JMAP, type: 'Email' });
|
||||
expect(cursor?.state).toBe('s2');
|
||||
expect(cursor?.consecutiveFailures).toBe(3);
|
||||
// A patch must not be able to rewrite `state` - only advance/seed can.
|
||||
store.transaction(() => {
|
||||
store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { drainPending: true });
|
||||
});
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('s2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('envelope tier', () => {
|
||||
it('does NOT reset has_body on an idempotent replay', () => {
|
||||
// Otherwise a replayed page looks like "body missing" to the backfill job and
|
||||
// re-downloads every body in the page.
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{}}'); });
|
||||
expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10)).toHaveLength(0);
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 2); });
|
||||
expect(
|
||||
store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10),
|
||||
'a replayed envelope upsert must not clear has_body',
|
||||
).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('patches only the two mutable properties, and no-ops for an absent id', () => {
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
const ok = store.transaction(() =>
|
||||
store.patchEnvelopeMutable(JMAP, 'e1', { keywordsJson: '{"$seen":true}', mailboxIds: ['archive'] }),
|
||||
);
|
||||
expect(ok).toBe(true);
|
||||
expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual(['archive']);
|
||||
// An update for an id we do not hold must leave no membership rows behind.
|
||||
const missing = store.transaction(() =>
|
||||
store.patchEnvelopeMutable(JMAP, 'nope', { keywordsJson: '{}', mailboxIds: ['inbox'] }),
|
||||
);
|
||||
expect(missing).toBe(false);
|
||||
expect(store.mailboxIdsFor(JMAP, 'nope')).toEqual([]);
|
||||
});
|
||||
|
||||
it('never writes a body whose envelope is gone', () => {
|
||||
// A body fetched moments before its envelope was destroyed in the same cycle
|
||||
// would otherwise land as an orphan.
|
||||
const wrote = store.transaction(() => store.putBodyIfEnvelopeExists(JMAP, 'ghost', '{}'));
|
||||
expect(wrote).toBe(false);
|
||||
expect(store.getBody(JMAP, 'ghost')).toBeNull();
|
||||
});
|
||||
|
||||
it('deleting an email takes its body, membership and queue row with it', () => {
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1);
|
||||
store.enqueueBodies([{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }]);
|
||||
});
|
||||
store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"a":1}'); });
|
||||
store.transaction(() => { store.deleteEmails(JMAP, ['e1']); });
|
||||
expect(store.getBody(JMAP, 'e1')).toBeNull();
|
||||
expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual([]);
|
||||
expect(store.countWantedBodies(JMAP, Date.now())).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the reconcile sweep', () => {
|
||||
it('refuses to run without a pinned stamp rather than deleting unverified records', () => {
|
||||
expect(() => store.sweep(JMAP, '2026-01-01T00:00:00.000Z', undefined))
|
||||
.toThrow(/refusing to delete unverified/);
|
||||
});
|
||||
|
||||
it('keeps what the enumeration re-saw and deletes what it did not', () => {
|
||||
// The whole "seen set as one integer" trick: re-upserting refreshes
|
||||
// cached_at, and the sweep deletes anything still below the pin.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('kept', '2026-08-01T00:00:00.000Z'),
|
||||
envelope('gone', '2026-08-02T00:00:00.000Z'),
|
||||
], 100);
|
||||
});
|
||||
const stamp = Math.max(500, store.maxEnvelopeCachedAt(JMAP) + 1);
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp);
|
||||
});
|
||||
store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); });
|
||||
expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull();
|
||||
expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull();
|
||||
});
|
||||
|
||||
it('a stamp taken from a FROZEN clock would sweep nothing; the derived one works', () => {
|
||||
// Both halves matter, and both fail silently. Exercising the real
|
||||
// `reconcileStamp` rather than re-deriving it in the test is the point.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('kept', '2026-08-01T00:00:00.000Z'),
|
||||
envelope('gone', '2026-08-02T00:00:00.000Z'),
|
||||
], 9_999);
|
||||
});
|
||||
const frozenNow = 1_000;
|
||||
|
||||
// The naive version: with the clock behind the data, nothing is below the
|
||||
// stamp, so a re-verified store sweeps zero rows and stale records live on.
|
||||
expect(store.sweep(JMAP, '2026-07-01T00:00:00.000Z', frozenNow)).toBe(0);
|
||||
|
||||
const stamp = reconcileStamp(frozenNow, store.maxEnvelopeCachedAt(JMAP));
|
||||
expect(stamp).toBe(10_000);
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp);
|
||||
});
|
||||
expect(store.transaction(() => store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp))).toBe(1);
|
||||
expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull();
|
||||
expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull();
|
||||
});
|
||||
|
||||
it('stamping an enumeration with `now` instead of the pin deletes what it just verified', () => {
|
||||
// The other direction of the same bug: the pin EXCEEDS now, so a page that
|
||||
// stamps with `now` lands below the pin and the sweep eats it.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], 9_999);
|
||||
});
|
||||
const now = 1_000;
|
||||
const stamp = reconcileStamp(now, store.maxEnvelopeCachedAt(JMAP));
|
||||
// Re-verified against the server, but stamped with the WRONG value.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], now);
|
||||
});
|
||||
store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); });
|
||||
expect(
|
||||
store.getEnvelopeRaw(JMAP, 'verified'),
|
||||
'this is the failure mode the pinned stamp exists to prevent',
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the body queue - the durable-terminal-state fixes', () => {
|
||||
it('enqueueBodies reports rows ACTUALLY INSERTED, not attempted', () => {
|
||||
// Reporting the attempted count made the mobile engine believe there was
|
||||
// unfinished work every cycle for as long as any envelope lacked a body,
|
||||
// chaining a new cycle every few seconds indefinitely.
|
||||
const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 };
|
||||
expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(1);
|
||||
expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(0);
|
||||
});
|
||||
|
||||
it('never resets attempts on a re-enqueue', () => {
|
||||
const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 };
|
||||
store.transaction(() => { store.enqueueBodies([entry]); });
|
||||
store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 0, 'boom'); });
|
||||
store.transaction(() => { store.enqueueBodies([entry]); });
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())[0]?.attempts).toBe(1);
|
||||
});
|
||||
|
||||
it('THE H1 REGRESSION: a gave-up row is KEPT and is never revived by a re-enqueue', () => {
|
||||
// Deleting the row on give-up was not enough: the backfill driver is
|
||||
// "envelope with no body", which cannot tell "not fetched yet" from
|
||||
// "deliberately not kept", so the next pass re-inserted a fresh attempts=0
|
||||
// row and a permanently-failing body was retried five times per cycle forever.
|
||||
store.transaction(() => {
|
||||
store.markBodyGaveUp(JMAP, [
|
||||
{ emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' },
|
||||
]);
|
||||
});
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['e1']);
|
||||
// Not WANTED any more, so the drain never picks it up again.
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0);
|
||||
// And a re-enqueue cannot resurrect it.
|
||||
const inserted = store.transaction(() =>
|
||||
store.enqueueBodies([
|
||||
{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
|
||||
]),
|
||||
);
|
||||
expect(inserted).toBe(0);
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('THE H1c REGRESSION: a cap-shed body is markable even with NO existing queue row', () => {
|
||||
// The download/discard loop: the cap sheds a body that was fetched and stored
|
||||
// successfully, so there is no queue row left to UPDATE. If the mark is
|
||||
// silently dropped, the envelope is still inside the body WINDOW, the backfill
|
||||
// re-enqueues it, it downloads again, and the cap sheds it again - unbounded
|
||||
// data use that never terminates.
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{"1":{"value":"x"}}}'); });
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0); // no queue row exists
|
||||
|
||||
store.transaction(() => {
|
||||
store.deleteBodies(JMAP, ['e1']);
|
||||
store.markBodyGaveUp(JMAP, [
|
||||
{ emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' },
|
||||
]);
|
||||
});
|
||||
expect(
|
||||
store.listBodyGiveUps(JMAP, 10),
|
||||
'the cap-shed mark must be an upsert, or the shed/re-download loop stays open',
|
||||
).toEqual(['e1']);
|
||||
// The envelope is back to has_body=0 and still in the window, so without the
|
||||
// mark the backfill WOULD pick it up. With the mark it is excluded.
|
||||
expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10).map((e) => e.id))
|
||||
.toEqual(['e1']);
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toContain('e1');
|
||||
});
|
||||
|
||||
it('clearing give-ups DELETES them, so they look like "never queued"', () => {
|
||||
// A cleared give-up must come back with a clean attempt count, which an
|
||||
// un-flag would not give.
|
||||
store.transaction(() => {
|
||||
store.markBodyGaveUp(JMAP, [
|
||||
{ emailId: 'a', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' },
|
||||
{ emailId: 'b', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' },
|
||||
]);
|
||||
});
|
||||
store.transaction(() => { store.clearBodyGiveUps(JMAP, 'shed-by-cap'); });
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['a']);
|
||||
store.transaction(() => { store.clearBodyGiveUps(JMAP); });
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toEqual([]);
|
||||
expect(
|
||||
store.transaction(() =>
|
||||
store.enqueueBodies([
|
||||
{ emailId: 'a', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
|
||||
]),
|
||||
),
|
||||
'a cleared give-up must be re-enqueueable',
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it('honours a backoff window', () => {
|
||||
store.transaction(() => {
|
||||
store.enqueueBodies([
|
||||
{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
|
||||
]);
|
||||
});
|
||||
store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 10_000, 'later'); });
|
||||
expect(store.takeBodyQueue(JMAP, 10, 5_000)).toHaveLength(0);
|
||||
expect(store.takeBodyQueue(JMAP, 10, 20_000)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('eviction', () => {
|
||||
it('cap eviction takes the oldest bodies first and leaves envelopes alone', () => {
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('old', '2026-01-01T00:00:00.000Z'),
|
||||
envelope('new', '2026-08-01T00:00:00.000Z'),
|
||||
], 1);
|
||||
});
|
||||
store.transaction(() => {
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'old', '{"v":"old"}');
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'new', '{"v":"new"}');
|
||||
});
|
||||
expect(store.oldestBodies(JMAP, 1).map((b) => b.emailId)).toEqual(['old']);
|
||||
store.transaction(() => { store.deleteBodies(JMAP, ['old']); });
|
||||
// The message stays LISTED - only its content went.
|
||||
expect(store.getEnvelopeRaw(JMAP, 'old')).not.toBeNull();
|
||||
expect(store.countBodies(JMAP)).toBe(1);
|
||||
});
|
||||
|
||||
it('no deletion path leaves an orphan body behind', () => {
|
||||
// This is the real invariant. `orphanBodies()` is a belt-and-braces sweep for
|
||||
// orphans a CRASH between two transactions could leave; it is deliberately
|
||||
// not reachable through the store's own API, which is what this asserts.
|
||||
// (So the detection query itself is covered only by the integration run, not
|
||||
// by this file - stated rather than papered over with a vacuous assertion.)
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('a', '2026-01-01T00:00:00.000Z'),
|
||||
envelope('b', '2026-08-01T00:00:00.000Z'),
|
||||
], 1);
|
||||
});
|
||||
store.transaction(() => {
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'a', '{"v":1}');
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'b', '{"v":2}');
|
||||
});
|
||||
expect(store.countBodies(JMAP)).toBe(2);
|
||||
|
||||
store.transaction(() => { store.deleteEmails(JMAP, ['a']); });
|
||||
expect(store.orphanBodies(JMAP, 10)).toEqual([]);
|
||||
|
||||
store.transaction(() => { store.evictEnvelopesBelow(JMAP, '2026-09-01T00:00:00.000Z'); });
|
||||
expect(store.countEnvelopes(JMAP)).toBe(0);
|
||||
expect(store.countBodies(JMAP)).toBe(0);
|
||||
expect(store.orphanBodies(JMAP, 10)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('purge', () => {
|
||||
it('purgeAll takes the CURSORS with the records', () => {
|
||||
// A record wipe that leaves a live cursor behind is the one state no amount
|
||||
// of syncing repairs: /changes cannot re-deliver mail that already existed
|
||||
// when the cursor was captured.
|
||||
seed(store);
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.transaction(() => { store.purgeAll(); });
|
||||
expect(store.countEnvelopes(JMAP)).toBe(0);
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull();
|
||||
expect(store.getCoverage(JMAP)).toBeNull();
|
||||
});
|
||||
|
||||
it('a wrong key is treated as unreadable and rebuilt, never as a prompt', () => {
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.close();
|
||||
const other = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key: randomBytes(32) });
|
||||
try {
|
||||
expect(other.countEnvelopes(JMAP)).toBe(0);
|
||||
} finally {
|
||||
other.close();
|
||||
}
|
||||
store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
|
||||
});
|
||||
});
|
||||
|
||||
describe('policy', () => {
|
||||
it('round-trips and clamps', () => {
|
||||
store.transaction(() => { store.setPolicy({ envelopeDays: 99999, bodyDays: 0, maxBodyMB: 1 }); });
|
||||
const policy = store.getPolicy();
|
||||
expect(policy.envelopeDays).toBe(3650);
|
||||
expect(policy.bodyDays).toBe(1);
|
||||
expect(policy.maxBodyMB).toBe(16);
|
||||
});
|
||||
|
||||
it('defaults when nothing was ever written', () => {
|
||||
expect(store.getPolicy()).toEqual(DEFAULT_POLICY);
|
||||
});
|
||||
});
|
||||
|
||||
describe('read path', () => {
|
||||
it('lists a mailbox page newest-first with a correct total', () => {
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('a', '2026-08-01T00:00:00.000Z'),
|
||||
envelope('b', '2026-08-02T00:00:00.000Z'),
|
||||
envelope('c', '2026-08-03T00:00:00.000Z', { mailboxIds: ['archive'] }),
|
||||
], 1);
|
||||
});
|
||||
const inbox = store.listEnvelopes(JMAP, 'inbox', 10, 0);
|
||||
expect(inbox.total).toBe(2);
|
||||
expect(inbox.rows.map((r) => String(r.id))).toEqual(['b', 'a']);
|
||||
// A null mailbox is "everything", which is what the unified views want.
|
||||
expect(store.listEnvelopes(JMAP, null, 10, 0).total).toBe(3);
|
||||
expect(store.listEnvelopes(JMAP, 'archive', 10, 0).rows.map((r) => String(r.id))).toEqual(['c']);
|
||||
});
|
||||
|
||||
it('reports the account ids it holds without needing a network session', () => {
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('a', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
expect(store.knownJmapAccountIds()).toEqual([JMAP]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('clampPolicy', () => {
|
||||
it('never lets the body window exceed the envelope window', () => {
|
||||
expect(clampPolicy({ envelopeDays: 30, bodyDays: 365, maxBodyMB: 100 }).bodyDays).toBe(30);
|
||||
});
|
||||
|
||||
it('falls back to defaults for junk input', () => {
|
||||
expect(clampPolicy({ envelopeDays: NaN } as never).envelopeDays).toBe(DEFAULT_POLICY.envelopeDays);
|
||||
expect(clampPolicy(null)).toEqual(DEFAULT_POLICY);
|
||||
expect(clampPolicy(undefined)).toEqual(DEFAULT_POLICY);
|
||||
});
|
||||
});
|
||||
|
||||
function seed(store: ReplicaStore): void {
|
||||
store.transaction(() => {
|
||||
for (const type of ['Email', 'Mailbox'] as const) {
|
||||
store.seedCursor(
|
||||
{ jmapAccountId: JMAP, type },
|
||||
mintEnumerationCommitment({
|
||||
jmapAccountId: JMAP,
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: '2026-01-01T00:00:00.000Z',
|
||||
sweepFloor: '2026-01-01T00:00:00.000Z',
|
||||
kind: 'bootstrap',
|
||||
}),
|
||||
1000,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user