diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05a84df6..1cde7cea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,80 +1,30 @@ # GitLab-CI dev→prod pipeline for VNCmail+ — GitOps via ArgoCD. # -# Revised after direct inspection of the real infrastructure found ArgoCD -# already installed (idle, zero Applications) on the dev-k8s-1/2/3 cluster. -# That's more idiomatic than a runner-executes-kubectl design, and it means -# this pipeline needs ZERO cluster credentials — CI only ever talks to the -# container registry and to this git repo. ArgoCD (which already has -# whatever cluster access it needs, set up once when its Applications were -# registered — see deploy/argocd/) is what actually applies anything. -# # Design: -# - One image name, environment lives only in the tag. No more -dev/-beta -# name confusion from the old GitHub Actions workflow. # - MR into `dev`: verify only (typecheck/lint/unit test/build check). No # push, no deploy — this is the multi-developer merge gate. -# - Push to `dev`: build+push an immutable `sha-` tag, then commit a -# one-line tag-bump into overlays/dev/image-tag/kustomization.yaml -# (`[skip ci]`, so this doesn't retrigger itself). ArgoCD's `vncmail-dev` -# Application has automated sync — it notices the git change and applies -# it. No approval needed, dev always deploys, and this job never touches -# the cluster directly. -# - Push to `main`: NEVER rebuilds. `main` only ever advances via +# - Push to `dev`: build+push an immutable `sha-` tag with Docker + +# docker-in-docker, then commit a one-line tag-bump into +# overlays/dev/image-tag/kustomization.yaml (`[skip ci]`). ArgoCD's +# `vncmail-dev` Application syncs it automatically. +# - Push to `main`: NEVER rebuilds. `main` only advances via # `git merge --ff-only dev`, so main's HEAD commit already has a built -# image (the same sha- tag dev already deployed). This job just bumps -# overlays/prod/image-tag/kustomization.yaml to point at that same tag. -# The actual promotion gate is a HUMAN clicking Sync on the `vncmail-prod` ArgoCD -# Application (deliberately NOT automated sync) — not a GitLab manual -# job, since ArgoCD already provides that exact gate more directly. -# Until prod Stalwart/hostname/secrets are real (see VNCMAIL-SETUP.md), -# nobody should click that Sync button — but nothing here does it for -# you either way. +# image. This job just bumps overlays/prod/image-tag/kustomization.yaml +# to point at that same tag. The actual promotion gate is a HUMAN +# clicking Sync on the `vncmail-prod` ArgoCD Application. # # Deliberately single-platform (linux/amd64) — this pipeline serves two # known amd64 microk8s clusters, not public multi-arch distribution (that's # what the GHCR release workflows are for, untouched by this file). # -# Registry history (so nobody re-litigates this from scratch). GitLab's own -# Container Registry was tried twice and does not work on this server: -# -# Round 1: registry_external_url was unset, so $CI_REGISTRY was empty and -# docker login silently fell through to Docker Hub. -# Round 2: after the server-side config, the project's sidebar DID show -# "Container Registry" and docker login DID succeed - but the push -# failed 403. Diagnosed 2026-08-05 by curling the vhost directly: -# -# $ curl -i https://registry.gitlab.vnc.biz/v2/ -# www-authenticate: Bearer realm="http://gitlab.vnc.biz/jwt/auth", -# service="dependency_proxy" -# x-runtime: 0.020470 -# x-gitlab-meta: {"correlation_id":...} -# -# x-runtime/x-gitlab-meta are RAILS headers, and the service is -# "dependency_proxy" - so nginx routes that hostname to the GitLab -# Rails app, which reads /v2/ as the dependency proxy (a Docker Hub -# pull-through cache), NOT to the registry container. The registry -# service was never actually wired behind that vhost. That is why -# login worked (Rails issues an unscoped dependency-proxy token) -# while a scoped :push request 403'd - the dependency proxy has no -# push concept at all. -# -# Fixing that is an nginx/omnibus change on the GitLab server (registry -# service must actually listen behind registry.gitlab.vnc.biz), not something -# any .gitlab-ci.yml can reach. Until someone does that, GHCR it is - the -# same image the sandbox already pulls, and confirmed public so the cluster -# needs no imagePullSecrets (see deploy/k8s/base/deployment.yaml). -# -# Prerequisite this file assumes (documented in VNCMAIL-SETUP.md, not -# something this file can set up itself): -# - A GitHub PAT with `write:packages` for ghcr.io/brvncde-dotcom as -# $GITLAB_CI_GHCR_TOKEN, plus the matching GitHub username as -# $GITLAB_CI_GHCR_USER — both masked+protected CI/CD variables. A GitHub -# credential can only come from GitHub; nothing GitLab-native substitutes. -# - A GitLab Runner (any kind — no cluster access needed at all now). +# Prerequisite this file assumes: +# - A GitLab Runner with Docker-in-Docker service support (Kubernetes or +# Docker executor). The `docker:28.4.0-dind` service requires privileged +# mode on most Kubernetes executors. # - Either "allow this job token to push to this project" enabled # (Settings → CI/CD → Job token permissions), OR a project access token -# with `write_repository` scope in $GITLAB_PUSH_TOKEN. The job below -# tries CI_JOB_TOKEN first (see the script). +# with `write_repository` scope in $GITLAB_PUSH_TOKEN. The bump jobs +# try CI_JOB_TOKEN first (see the script). # # deploy/k8s/ca/ (the EJBCA internal CA) is never referenced anywhere below, # and neither ArgoCD Application in deploy/argocd/ points at it — that stays @@ -87,11 +37,14 @@ stages: - bump-prod variables: - # GHCR, not GitLab's own registry - see the "Registry history" note in the - # header for the curl output proving why. Same image the sandbox already - # pulls today. - IMAGE: ghcr.io/brvncde-dotcom/vncmail-plus-dev + IMAGE: $CI_REGISTRY_IMAGE GIT_STRATEGY: clone + DOCKER_DRIVER: overlay2 + # DinD service is reached at the `docker` alias (set explicitly on the + # service below), not localhost. TLS disabled so the daemon listens on + # plaintext 2375 — same pattern as the working vnc-localidp pipeline. + DOCKER_HOST: tcp://docker:2375 + DOCKER_TLS_CERTDIR: "" # --------------------------------------------------------------------------- # verify — required check on every MR into dev. No registry, no cluster. @@ -118,45 +71,24 @@ verify: # --------------------------------------------------------------------------- build: stage: build - # Kaniko builds OCI images without a Docker daemon, so it needs neither a - # dind service nor a privileged pod — GitLab's own recommended approach - # for the Kubernetes executor specifically. The docker:27-cli + dind - # combination that was here before this got as far as a successful - # $CI_REGISTRY login, then failed every way it was pointed - # (unix:///var/run/docker.sock, tcp://docker:2375, tcp://localhost:2375): - # the dind container itself was never actually listening, which on this - # executor means it needs `privileged: true` in the runner's own - # config.toml — a cluster/GitLab-admin setting outside this file's - # control. Kaniko sidesteps that requirement entirely rather than chasing - # runner permissions further, and is also the safer default on a shared - # cluster (no privileged containers at all). - image: - name: gcr.io/kaniko-project/executor:v1.23.2-debug - entrypoint: [""] + image: docker:28.4.0 + services: + - name: docker:28.4.0-dind + alias: docker rules: - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"' + before_script: + - until docker info; do sleep 1; done + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" script: - # Fail loudly and immediately if the GHCR credentials aren't configured, - # rather than letting kaniko get all the way through a full Next.js build - # and only then 403 on push (which is exactly how the GitLab-registry - # attempt burned several pipeline runs). - - | - if [ -z "$GITLAB_CI_GHCR_TOKEN" ] || [ -z "$GITLAB_CI_GHCR_USER" ]; then - echo "ERROR: \$GITLAB_CI_GHCR_TOKEN and/or \$GITLAB_CI_GHCR_USER are not set." - echo "Add both under Settings -> CI/CD -> Variables (masked + protected)." - echo "The token is a GitHub PAT with the write:packages scope." - exit 1 - fi - - mkdir -p /kaniko/.docker - - | - echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"$(printf '%s:%s' "$GITLAB_CI_GHCR_USER" "$GITLAB_CI_GHCR_TOKEN" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json - > - /kaniko/executor - --context "$CI_PROJECT_DIR" - --dockerfile "$CI_PROJECT_DIR/Dockerfile" + docker build --build-arg GIT_COMMIT=$CI_COMMIT_SHA - --destination "$IMAGE:sha-$CI_COMMIT_SHORT_SHA" - --destination "$IMAGE:dev-latest" + -t "$IMAGE:sha-$CI_COMMIT_SHORT_SHA" + -t "$IMAGE:dev-latest" + . + - docker push "$IMAGE:sha-$CI_COMMIT_SHORT_SHA" + - docker push "$IMAGE:dev-latest" # --------------------------------------------------------------------------- # bump-dev — no cluster access. Commits the just-built tag into the overlay @@ -164,7 +96,9 @@ build: # --------------------------------------------------------------------------- bump-dev: stage: bump-dev - image: alpine/git:2.47.0 + # alpine/git:2.47.0 was never published on Docker Hub — the 2.47.x line + # starts at 2.47.1. Using 2.47.2 (latest 2.47.x). + image: alpine/git:2.47.2 rules: - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"' script: @@ -176,7 +110,7 @@ bump-dev: apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev + - name: vncmail-plus newName: $IMAGE newTag: $TAG EOF @@ -199,7 +133,7 @@ bump-dev: # --------------------------------------------------------------------------- bump-prod: stage: bump-prod - image: alpine/git:2.47.0 + image: alpine/git:2.47.2 rules: - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"' script: @@ -215,7 +149,7 @@ bump-prod: apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev + - name: vncmail-plus newName: $IMAGE newTag: $TAG EOF diff --git a/app/api/offline/search/route.ts b/app/api/offline/search/route.ts index eb938050..3de06365 100644 --- a/app/api/offline/search/route.ts +++ b/app/api/offline/search/route.ts @@ -79,7 +79,12 @@ export async function GET(request: NextRequest) { try { const stats = index.stats(); if (!query) return { hits: [] as SearchHit[], stats }; - return { hits: index.search({ query, types, limit }), stats: wantStats ? stats : undefined }; + // 'any': this route is the AI/RAG retrieval surface (see module + // header) - its one real caller sends natural-language questions, + // not deliberate search-box keywords, so strict AND-every-token + // matching (the default) drops nearly all of them. See + // toFtsMatchQueryAny's docstring for the confirmed-live failure. + return { hits: index.search({ query, types, limit, mode: 'any' }), stats: wantStats ? stats : undefined }; } finally { index.close(); } diff --git a/components/settings/ai-assistant-settings.tsx b/components/settings/ai-assistant-settings.tsx index a9d104fc..c834f86c 100644 --- a/components/settings/ai-assistant-settings.tsx +++ b/components/settings/ai-assistant-settings.tsx @@ -1,7 +1,7 @@ 'use client'; import { useCallback, useEffect, useMemo, useState } from 'react'; -import { RefreshCw, CheckCircle, AlertTriangle, Loader2, Plus, Trash2 } from 'lucide-react'; +import { RefreshCw, CheckCircle, AlertTriangle, Loader2, Plus, Trash2, Sparkles, X } from 'lucide-react'; import { SettingsSection, SettingItem, ToggleSwitch, RadioGroup, Select } from './settings-section'; import { Button } from '@/components/ui/button'; import { apiFetch } from '@/lib/browser-navigation'; @@ -9,6 +9,14 @@ import { DEFAULT_AI_POLICY, type AiPolicy } from '@/lib/ai/types'; import { supportsLocalLlm, localLlmNeedsCorsSetup } from '@/lib/platform-capabilities'; import { getAiApiKey, setAiApiKey, clearAiApiKey } from '@/lib/ai/key-store'; import { loadAiSettings, saveAiSettings, createProfile, type AiLocalSettings } from '@/lib/ai/local-settings'; +import { + discoverLocalOllama, + recommendDefaultModel, + largestModel, + isLocalDiscoveryDismissed, + dismissLocalDiscovery, + type LocalDiscoveryResult, +} from '@/lib/ai/local-discovery'; import { askMail, listLocalModels, @@ -91,6 +99,47 @@ export function AiAssistantSettings() { } }, [settings.localBaseUrl]); + // ── Local discovery — proactively find an already-running Ollama and + // offer a one-click connect, rather than making the user hunt down and + // type a base URL + model name by hand. ── + const [discovery, setDiscovery] = useState(null); + const [discoveryDismissed, setDiscoveryDismissed] = useState(true); + + useEffect(() => { + setDiscoveryDismissed(isLocalDiscoveryDismissed()); + }, []); + + useEffect(() => { + if (!canUseLocal || discoveryDismissed || settings.localModel) return; + let cancelled = false; + (async () => { + const result = await discoverLocalOllama(); + if (!cancelled) setDiscovery(result); + })(); + return () => { + cancelled = true; + }; + }, [canUseLocal, discoveryDismissed, settings.localModel]); + + const connectDiscoveredLocal = useCallback(() => { + if (!discovery) return; + const recommended = recommendDefaultModel(discovery.models) ?? discovery.models[0]?.name ?? null; + if (!recommended) return; + setSettings((prev) => { + const next: AiLocalSettings = { ...prev, provider: 'local', localBaseUrl: discovery.baseUrl, localModel: recommended }; + saveAiSettings(next); + return next; + }); + setLocalModels(discovery.models.filter((m) => m.capabilities.includes('completion')).map((m) => m.name)); + setDiscovery(null); + }, [discovery]); + + const dismissDiscoveryBanner = useCallback(() => { + dismissLocalDiscovery(); + setDiscoveryDismissed(true); + setDiscovery(null); + }, []); + // ── Server provider ── const [serverModels, setServerModels] = useState([]); const [refreshingServer, setRefreshingServer] = useState(false); @@ -209,8 +258,35 @@ export function AiAssistantSettings() { ); } + const discoveryRecommended = discovery ? recommendDefaultModel(discovery.models) : null; + const discoveryLargest = discovery ? largestModel(discovery.models) : null; + return (
+ {discovery && discoveryRecommended && ( +
+ +
+

Local AI found on this machine

+

+ Ollama is running at {discovery.baseUrl} with {discovery.models.length} model{discovery.models.length === 1 ? '' : 's'} installed. + Recommended for quick answers: {discoveryRecommended}. + {discoveryLargest && discoveryLargest !== discoveryRecommended && ( + <> Also available for higher-quality answers: {discoveryLargest}. + )} +

+
+ + +
+
+
+ )} + ` per deploy, moving pointers `dev-latest`/`prod-latest`). The -`ghcr.io/brvncde-dotcom/vncmail-plus-dev` image referenced in `base/deployment.yaml` -is a legacy default only — CI overrides it per-deploy via `kubectl set image`, -so what's committed there never needs to track what's actually running. +(tag `sha-` per deploy, moving pointer `dev-latest`). The generic `vncmail-plus` +image name in `base/deployment.yaml` is a placeholder — kustomize's image-tag +Component replaces it with the real registry path on every deploy. --- @@ -98,15 +97,15 @@ pick up the fix): ```bash cd deploy/k8s/overlays/dev # or overlays/prod, once real -# a) Image-pull secret — the registry package is private. -kubectl create secret docker-registry ghcr-pull \ +# a) Image-pull secret — the GitLab registry requires authentication. +# Use a project deploy token with `read_registry` scope, or the CI job +# token (short-lived — better for CI, not for long-running clusters). +kubectl create secret docker-registry gitlab-registry \ --namespace vncmail \ - --docker-server=ghcr.io \ - --docker-username=brvncde-dotcom \ - --docker-password='' \ - --docker-email=br@vnc.biz -# Once CI has cut over to registry.gitlab.vnc.biz, this becomes a -# docker-registry secret for that registry instead — see VNCMAIL-SETUP.md. + --docker-server=registry.gitlab.vnc.biz \ + --docker-username= \ + --docker-password='' \ + --docker-email=ci@vnc.biz # b) App config secret — copy the template, set a real SESSION_SECRET, apply. cp secret.example.yaml secret.yaml @@ -117,8 +116,8 @@ kubectl apply -f secret.yaml kubectl apply -k . ``` -> Alternative to (a): make the registry package public, then delete the -> `imagePullSecrets:` block from `base/deployment.yaml`. +> Alternative to (a): make the GitLab container registry public for this +> project, then delete the `imagePullSecrets:` block from `base/deployment.yaml`. After this one-time setup, routine deploys to `dev` happen automatically via CI on every push — see "Routine deploys go through CI now" above. This @@ -170,7 +169,7 @@ ArgoCD re-sync. | Symptom | Cause / fix | |---------|-------------| -| Pod `ImagePullBackOff` | `ghcr-pull` secret missing/expired, or package still private. Recreate the secret (§3a) or make the package public. | +| Pod `ImagePullBackOff` | `gitlab-registry` secret missing/expired, or token lacks `read_registry`. Recreate the secret (§3a) or make the registry public. | | Pod `CrashLoopBackOff`, logs show `EACCES`/permission on `/app/data` | Volume not writable by uid 1001. `securityContext.fsGroup: 1001` is set in `base/deployment.yaml` — keep it; some storage drivers also need it on the PVC. | | PVC stuck `Pending` | Wrong `storageClassName` in `base/pvc.yaml`. Set it to one from `kubectl get sc`. | | Ingress has no address / no cert | Wrong `ingressClassName` or cert issuer. Match bulwark's (§2). Check `kubectl -n vncmail describe ingress vncmail-plus`. | diff --git a/deploy/k8s/base/deployment.yaml b/deploy/k8s/base/deployment.yaml index b0ad23c0..cc41423c 100644 --- a/deploy/k8s/base/deployment.yaml +++ b/deploy/k8s/base/deployment.yaml @@ -23,20 +23,18 @@ spec: fsGroup: 1001 runAsUser: 1001 runAsGroup: 1001 - # Confirmed 2026-08-05: ghcr.io/brvncde-dotcom/vncmail-plus-dev IS public - # (anonymous token pull succeeded) — no imagePullSecrets needed. This is - # deploy/k8s/README.md's own documented alternative to creating a - # ghcr-pull secret. Removed rather than left referencing a - # not-yet-created secret, which would otherwise block every pod from - # starting regardless of the image being public (kubelet fails to - # resolve a missing imagePullSecrets entry before it ever gets to - # deciding whether auth was actually required). + # The GitLab container registry is private by default. Nodes need a + # docker-registry secret named `gitlab-registry` in the target namespace. + # Create it once per environment during first-time setup + # (see deploy/k8s/README.md §3a). + imagePullSecrets: + - name: gitlab-registry containers: - name: vncmail-plus - # Default/legacy value — CI overrides the image per-deploy via - # `kustomize edit set image`, so what's committed here never goes - # stale. For a one-off manual apply, pin a digest instead of :latest. - image: ghcr.io/brvncde-dotcom/vncmail-plus-dev:latest + # Generic placeholder — the real image name + tag are injected by the + # image-tag kustomize Component on every deploy (see + # overlays/*/image-tag/kustomization.yaml, rewritten by CI). + image: vncmail-plus:latest imagePullPolicy: Always ports: - containerPort: 3000 diff --git a/deploy/k8s/base/ingress.yaml b/deploy/k8s/base/ingress.yaml index 24dbb55e..c3c00482 100644 --- a/deploy/k8s/base/ingress.yaml +++ b/deploy/k8s/base/ingress.yaml @@ -15,6 +15,7 @@ metadata: name: vncmail-plus annotations: cert-manager.io/cluster-issuer: CHANGEME + traefik.ingress.kubernetes.io/router.middlewares: traefik-redirect-to-https@kubernetescrd spec: ingressClassName: traefik tls: diff --git a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml index 0f6cd8a1..71772d4f 100644 --- a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml +++ b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml @@ -1,11 +1,8 @@ -# Owned by CI (the bump-dev job in .gitlab-ci.yml), not by hand. Kept as its -# own Component so CI only ever rewrites this 6-line file, never the parent -# overlays/dev/kustomization.yaml (structure/patches there stay under normal -# code review — CI regenerating a whole hand-maintained file on every push -# would silently revert any change made there between deploys). +# Owned by CI (bump-dev job in .gitlab-ci.yml) - regenerated every +# push to dev. Do not hand-edit; edits here get overwritten. apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev - newName: ghcr.io/brvncde-dotcom/vncmail-plus-dev - newTag: sha-147660a + - name: vncmail-plus + newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus + newTag: sha-2a8778c9 diff --git a/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml b/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml index 05011284..2203d87c 100644 --- a/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml +++ b/deploy/k8s/overlays/dev/patch-image-pull-policy.yaml @@ -4,19 +4,6 @@ # is pure waste - the content behind that tag can never change, so re-pulling # it on every pod start only adds a registry round-trip and a hard dependency # on the registry being reachable at scheduling time. -# -# It is also load-bearing right now: until CI can actually push (GitLab's -# registry vhost serves Rails, not the registry - see .gitlab-ci.yml's -# "Registry history" note), sha- tagged images are side-loaded straight into -# each node's containerd: -# -# docker save --platform linux/amd64 -o vncmail.tar : -# scp vncmail.tar dev-k8s-N:/tmp/ && ssh dev-k8s-N \ -# 'microk8s ctr images import /tmp/vncmail.tar' -# -# imported to ALL of dev-k8s-1/2/3 so the pod can schedule anywhere. With -# Always, kubelet would ignore that local image and fail on a registry pull -# for a tag the registry has never seen. apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/k8s/overlays/dev/patch-ingress.yaml b/deploy/k8s/overlays/dev/patch-ingress.yaml index f9efd41a..ff268a5f 100644 --- a/deploy/k8s/overlays/dev/patch-ingress.yaml +++ b/deploy/k8s/overlays/dev/patch-ingress.yaml @@ -1,14 +1,12 @@ -# dev-k8s cluster confirmed to have a `letsencrypt-staging` ClusterIssuer -# already (no `letsencrypt-prod` exists there) - staging avoids burning -# Let's Encrypt's real rate limits while this is still being stood up. -# vncmail.sandbox.vnc.de DNS does not point here yet either - this is the -# intended host, not a live one (see VNCMAIL-SETUP.md for what's still open). +# Dev now uses letsencrypt-prod so the sandbox certificate is trusted by +# browsers. Both letsencrypt-staging and letsencrypt-prod ClusterIssuers +# exist on the dev-k8s cluster (see VNCiAC-dev-cluster-runbook.md §7). apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: vncmail-plus annotations: - cert-manager.io/cluster-issuer: letsencrypt-staging + cert-manager.io/cluster-issuer: letsencrypt-prod spec: tls: - hosts: diff --git a/deploy/k8s/overlays/prod/image-tag/kustomization.yaml b/deploy/k8s/overlays/prod/image-tag/kustomization.yaml index 7619b621..d8191fe5 100644 --- a/deploy/k8s/overlays/prod/image-tag/kustomization.yaml +++ b/deploy/k8s/overlays/prod/image-tag/kustomization.yaml @@ -7,6 +7,6 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: - - name: ghcr.io/brvncde-dotcom/vncmail-plus-dev + - name: vncmail-plus newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus newTag: not-yet-promoted diff --git a/e2e/electron-ai-local-index.spec.ts b/e2e/electron-ai-local-index.spec.ts new file mode 100644 index 00000000..f2033dea --- /dev/null +++ b/e2e/electron-ai-local-index.spec.ts @@ -0,0 +1,199 @@ +import { test, expect, _electron as electron } from '@playwright/test'; +import type { ElectronApplication, Page } from '@playwright/test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +/** + * Proves the two hardest-to-fake claims about the AI Assistant's `local` + * class in the REAL packaged desktop shell, not a browser tab: + * + * 1. The LLM genuinely runs locally — a direct browser-side fetch to + * this machine's own Ollama (127.0.0.1:11434), never proxied through + * this app's backend. + * 2. It is genuinely grounded in the ENCRYPTED LOCAL SQLITE/FTS5 MAIL + * INDEX (lib/mail-index/**), not the separate real-JMAP-embeddings + * server leg (lib/ai/retrieval/mail-embeddings.ts) — AI_SERVER_BASE_URL + * is deliberately left UNSET here so only the local FTS leg can + * supply retrieval context. If this test passes, the local index + * leg is the only possible source of the grounded answer. + * + * Needs a real launch through electron/main.ts's startStandaloneServer(), + * not ELECTRON_LOAD_URL — that's the only code path that wires up the + * fd-3 key channel / safeStorage the encrypted index depends on (see + * integration/tests/12-electron-mail-index.spec.ts's header for the full + * reasoning). That function picks a random free port every launch, which + * would make it impossible to also point DEV_MOCK_JMAP's JMAP_SERVER_URL + * at this same server's own /api/dev-jmap route — hence + * VNCMAIL_TEST_FIXED_PORT, a narrow, off-by-default escape hatch added to + * electron/main.ts specifically to make this test possible without a real + * Stalwart fixture. + * + * Requires a real Ollama already running on this machine with at least one + * completion-capable model installed — skips (not fails) otherwise, since + * "no local LLM on this machine" is an environment fact, not a bug. + */ + +const projectRoot = path.resolve(__dirname, '..'); +const FIXED_PORT = 39217; +const ORIGIN = `http://127.0.0.1:${FIXED_PORT}`; + +async function ollamaIsUp(): Promise { + try { + const res = await fetch('http://127.0.0.1:11434/api/tags'); + if (!res.ok) return false; + const body = (await res.json()) as { models?: Array<{ capabilities?: string[] }> }; + return (body.models ?? []).some((m) => !m.capabilities || m.capabilities.includes('completion')); + } catch { + return false; + } +} + +test.describe('Electron desktop shell - local LLM answers from the real encrypted mail index', () => { + let electronApp: ElectronApplication; + let appWindow: Page; + let userDataDir: string; + + test.beforeAll(async () => { + if (!(await ollamaIsUp())) { + test.skip(true, 'No local Ollama with a completion-capable model reachable on this machine — environment fact, not a failure.'); + } + + userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-ai-index-test-')); + + electronApp = await electron.launch({ + args: [projectRoot, `--user-data-dir=${userDataDir}`], + env: { + ...process.env, + VNCMAIL_TEST_FIXED_PORT: String(FIXED_PORT), + DEV_MOCK_JMAP: 'true', + JMAP_SERVER_URL: `${ORIGIN}/api/dev-jmap`, + SESSION_SECRET: 'electron-ai-local-index-verify-32-chars-min', + // Deliberately UNSET: isolates grounding to the local FTS leg (see + // module header) — the server embeddings leg 404s cleanly instead + // of silently also being able to answer the question. + AI_SERVER_BASE_URL: '', + NODE_ENV: 'production', + }, + }); + + appWindow = await electronApp.firstWindow(); + await appWindow.waitForLoadState('domcontentloaded'); + }); + + test.afterAll(async () => { + await electronApp?.close(); + if (userDataDir) fs.rmSync(userDataDir, { recursive: true, force: true }); + }); + + test('logs in, builds the real encrypted index, and a local Ollama model answers a mail question grounded in it', async () => { + // ── 1. Real dev-mode login (sets the real session cookie the offline + // index and every other server-side-identity feature need). ── + const devLoginContainer = appWindow.locator('div', { hasText: 'Dev mode - logging in as dev@localhost' }).last(); + await devLoginContainer.getByRole('button').click(); + await appWindow.waitForURL((url) => !url.pathname.includes('login'), { timeout: 20000 }); + + // ── 2. Build the real encrypted local index: delta-sync the mock + // account's mail into the replica store, then write it into SQLite/FTS5. + // Chains /api/offline/sync while unfinishedWork is true, capped so a + // real bug can't hang the test forever. ── + const syncOutcome = await appWindow.evaluate(async () => { + let unfinished = true; + let calls = 0; + const statuses: number[] = []; + while (unfinished && calls < 10) { + const res = await fetch('/api/offline/sync', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' }); + statuses.push(res.status); + if (!res.ok) break; + const body = await res.json(); + unfinished = body.unfinishedWork === true; + calls++; + } + const reindexRes = await fetch('/api/offline/reindex', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ catchUp: true }) }); + return { syncStatuses: statuses, syncCalls: calls, reindexStatus: reindexRes.status, reindexBody: await reindexRes.json().catch(() => null) }; + }); + console.log('[ai-local-index] sync+reindex outcome:', JSON.stringify(syncOutcome)); + expect(syncOutcome.syncStatuses.every((s) => s === 200)).toBe(true); + expect(syncOutcome.reindexStatus).toBe(200); + + // ── 3. Prove the local index itself is real and queryable BEFORE + // touching the LLM at all — isolates "is the SQLite/FTS5 index working" + // from "did the model use it correctly". ── + const directSearch = await appWindow.evaluate(async () => { + const res = await fetch(`/api/offline/search?q=${encodeURIComponent('Villa sul Lago check-in')}&limit=6`); + return { status: res.status, body: await res.json().catch(() => null) }; + }); + console.log('[ai-local-index] direct /api/offline/search result:', JSON.stringify(directSearch.body)); + expect(directSearch.status, 'the encrypted local index must be reachable (200), not 404 (feature disabled) or 503 (no key channel)').toBe(200); + expect(directSearch.body?.ok).toBe(true); + const hitTitles = (directSearch.body?.hits ?? []).map((h: { title?: string }) => h.title ?? ''); + expect(hitTitles.some((t: string) => /villa sul lago/i.test(t)), `expected a "Villa sul Lago" hit in the real index, got: ${JSON.stringify(hitTitles)}`).toBe(true); + + // ── 4. Navigate to the real AI Assistant settings UI and use the + // local-discovery "Connect" banner — the exact flow a real user takes, + // proving discovery -> connect -> ask works as one integrated feature. + // Deliberately an in-app SPA navigation (click the real sidebar link), + // NOT appWindow.goto() — a full page reload drops whatever client-only + // session state the dev-mode login established (confirmed: goto('/settings') + // bounces straight back to /login even though the JMAP session cookie + // from step 2/3 is still valid), so the click is load-bearing, not + // cosmetic. ── + await appWindow.locator('a[href="/settings"], a[href*="/settings"]').first().click(); + const searchBox = appWindow.locator('input[type="search"]').first(); + await searchBox.fill('AI Assistant'); + await appWindow.getByRole('button', { name: 'AI Assistant' }).click(); + + const connectButton = appWindow.getByRole('button', { name: /Connect/i }); + await expect(connectButton, 'the local-discovery banner should appear since a real Ollama is running on this machine').toBeVisible({ timeout: 10000 }); + const bannerText = await appWindow.locator('text=Local AI found on this machine').locator('..').innerText(); + console.log('[ai-local-index] discovery banner text:', bannerText); + await connectButton.click(); + + // ── 5. Ask a question only answerable by combining the local LLM + // with the local index's actual content. ── + const questionBox = appWindow.getByPlaceholder(/What did legal say/i); + await questionBox.fill('When is check-in for the Villa sul Lago booking, and what time?'); + const askButton = appWindow.getByRole('button', { name: /^Ask$/ }); + await expect(askButton, 'Ask must be enabled immediately after Connect pre-fills provider+model').toBeEnabled({ timeout: 5000 }); + + const chatRequests: string[] = []; + const offlineSearchCalls: Array<{ url: string; status: number; body: unknown }> = []; + appWindow.on('request', (req) => { + if (req.url().includes('11434')) chatRequests.push(`${req.method()} ${req.url()}`); + }); + appWindow.on('response', async (res) => { + if (res.url().includes('/api/offline/search')) { + offlineSearchCalls.push({ url: res.url(), status: res.status(), body: await res.json().catch(() => null) }); + } + }); + + // Log the exact prompt Ollama actually received, straight from the + // request body — the ground truth for "did retrieval even fire". + const ollamaChatPayloads: unknown[] = []; + await appWindow.route('**/api/chat', async (route) => { + try { + ollamaChatPayloads.push(JSON.parse(route.request().postData() ?? 'null')); + } catch { /* ignore parse failure, still let the request through */ } + await route.continue(); + }); + + await askButton.click(); + + const answerLocator = appWindow.locator('p.whitespace-pre-wrap').first(); + await expect(answerLocator, 'the local Ollama model should produce an answer within a generous timeout').toBeVisible({ timeout: 60000 }); + const answerText = await answerLocator.innerText(); + console.log('[ai-local-index] final answer:', answerText); + console.log('[ai-local-index] direct-to-Ollama requests observed:', chatRequests); + console.log('[ai-local-index] /api/offline/search calls during Ask:', JSON.stringify(offlineSearchCalls)); + console.log('[ai-local-index] exact payload(s) sent to Ollama /api/chat:', JSON.stringify(ollamaChatPayloads)); + + // The real proof: the model's own words contain the fact that only + // exists in the indexed email (28 March, 15:00), and the request log + // shows the renderer talked to Ollama's loopback address directly. + expect(answerText).toMatch(/28\s*march|march\s*28/i); + expect(answerText).toMatch(/15:00|3\s*pm|3:00\s*pm/i); + expect(chatRequests.some((r) => r.includes('/api/chat')), `expected a direct renderer -> Ollama /api/chat request, saw: ${JSON.stringify(chatRequests)}`).toBe(true); + + await appWindow.screenshot({ path: path.join(projectRoot, 'electron-ai-local-index-result.png'), fullPage: true }); + }); +}); diff --git a/electron/main.ts b/electron/main.ts index b3c5fed8..56417d02 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -173,7 +173,18 @@ async function startStandaloneServer(): Promise { ); } - const port = await getFreePort(); + // Normally a random free port, chosen fresh every launch - JMAP_SERVER_URL + // never needs to reference it back (a real deployment's Stalwart lives at + // its own fixed address). VNCMAIL_TEST_FIXED_PORT is a narrow escape hatch + // for e2e tests that DO need to know the port ahead of time - specifically + // to point DEV_MOCK_JMAP's JMAP_SERVER_URL at this same standalone server's + // own /api/dev-jmap route, which is the only way to exercise the real + // encrypted offline index (lib/mail-index/**) without a real Stalwart + // fixture: that index's key channel only gets wired up in this function, + // never when ELECTRON_LOAD_URL bypasses it for a plain `next dev` target. + // Unset in every normal launch, so this changes nothing outside a test run. + const fixedPort = process.env.VNCMAIL_TEST_FIXED_PORT ? Number(process.env.VNCMAIL_TEST_FIXED_PORT) : null; + const port = fixedPort && Number.isInteger(fixedPort) ? fixedPort : await getFreePort(); const url = `http://127.0.0.1:${port}`; const storeDir = getIndexStoreDir(); diff --git a/lib/ai/__tests__/local-discovery.test.ts b/lib/ai/__tests__/local-discovery.test.ts new file mode 100644 index 00000000..b0e09eff --- /dev/null +++ b/lib/ai/__tests__/local-discovery.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; +import { + discoverLocalOllama, + recommendDefaultModel, + largestModel, + isLocalDiscoveryDismissed, + dismissLocalDiscovery, + type DiscoveredLocalModel, +} from '../local-discovery'; + +/** + * Real model list from this machine's Ollama (`curl 127.0.0.1:11434/api/tags`, + * 2026-08-06) — used as the test fixture rather than invented data, per the + * explicit instruction to use the real local runtime as the test case for + * "which queries are required and how to add most of the modules + * automatically". Sizes/params/capabilities are copied verbatim. + */ +const REAL_MACHINE_MODELS: DiscoveredLocalModel[] = [ + { name: 'nomic-embed-text:latest', capabilities: ['embedding'], parameterSize: '137M', sizeBytes: 274_302_450 }, + { name: 'qwen2.5:32b', capabilities: ['completion', 'tools'], parameterSize: '32.8B', sizeBytes: 19_851_349_669 }, + { name: 'gemma4:12b-mlx', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '', sizeBytes: 9_977_519_169 }, + { name: 'gemma4:latest', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '8.0B', sizeBytes: 9_608_350_718 }, + { name: 'deepseek-r1:32b', capabilities: ['completion', 'thinking'], parameterSize: '32.8B', sizeBytes: 19_851_337_809 }, + { name: 'deepseek-r1:14b', capabilities: ['completion', 'thinking'], parameterSize: '14.8B', sizeBytes: 8_988_112_209 }, + { name: 'llama3.2:latest', capabilities: ['completion', 'tools'], parameterSize: '3.2B', sizeBytes: 2_019_393_189 }, + { name: 'hermes3:8b', capabilities: ['completion', 'tools'], parameterSize: '8B', sizeBytes: 4_661_227_000 }, + { name: 'qwen3:latest', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '8.2B', sizeBytes: 5_200_000_000 }, + { name: 'gemma4:e4b', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '8.0B', sizeBytes: 9_600_000_000 }, + { name: 'qwen3.5:latest', capabilities: ['vision', 'completion', 'tools', 'thinking'], parameterSize: '9.7B', sizeBytes: 6_600_000_000 }, +]; + +describe('recommendDefaultModel', () => { + it('picks the smallest non-"thinking" chat model from a real mixed fleet', () => { + // llama3.2 (2.0GB) is the smallest completion-capable, non-reasoning + // model on this real machine — everything smaller is embedding-only. + expect(recommendDefaultModel(REAL_MACHINE_MODELS)).toBe('llama3.2:latest'); + }); + + it('never recommends an embedding-only model', () => { + const onlyEmbedding = [REAL_MACHINE_MODELS[0]]; // nomic-embed-text + expect(recommendDefaultModel(onlyEmbedding)).toBeNull(); + }); + + it('falls back to the smallest "thinking" model when nothing else qualifies', () => { + const onlyReasoning = REAL_MACHINE_MODELS.filter((m) => m.capabilities.includes('thinking') && !m.capabilities.includes('vision')); + // Smallest of the thinking-only pool here is qwen3 (5.2GB) before gemma4 variants. + expect(recommendDefaultModel(onlyReasoning)).toBe('qwen3:latest'); + }); + + it('returns null when no models are chat-capable at all', () => { + expect(recommendDefaultModel([])).toBeNull(); + }); +}); + +describe('largestModel', () => { + it('picks the biggest chat-capable model — qwen2.5:32b, by 11,860 bytes over deepseek-r1:32b', () => { + // Both are ~19.85GB on this real machine (same base size class), but + // qwen2.5:32b's actual manifest is very slightly larger — not a tie. + expect(largestModel(REAL_MACHINE_MODELS)).toBe('qwen2.5:32b'); + }); + + it('excludes embedding-only models even though they can be tiny or huge', () => { + expect(largestModel([REAL_MACHINE_MODELS[0]])).toBeNull(); + }); +}); + +describe('discoverLocalOllama', () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + vi.restoreAllMocks(); + }); + + it('parses a real-shaped /api/tags response into DiscoveredLocalModel[]', async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + models: [ + { name: 'llama3.2:latest', capabilities: ['completion', 'tools'], size: 2_019_393_189, details: { parameter_size: '3.2B' } }, + { name: 'nomic-embed-text:latest', capabilities: ['embedding'], size: 274_302_450, details: { parameter_size: '137M' } }, + ], + }), + }) as unknown as typeof fetch; + + const result = await discoverLocalOllama(['http://127.0.0.1:11434']); + expect(result).not.toBeNull(); + expect(result?.baseUrl).toBe('http://127.0.0.1:11434'); + expect(result?.models).toHaveLength(2); + expect(result?.models[0]).toEqual({ + name: 'llama3.2:latest', + capabilities: ['completion', 'tools'], + parameterSize: '3.2B', + sizeBytes: 2_019_393_189, + }); + }); + + it('requires exactly one query — a single /api/tags call, no follow-up /api/show requests', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ models: [{ name: 'llama3.2:latest', capabilities: ['completion'], size: 1, details: {} }] }), + }); + global.fetch = fetchMock as unknown as typeof fetch; + + await discoverLocalOllama(['http://127.0.0.1:11434']); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith('http://127.0.0.1:11434/api/tags', expect.anything()); + }); + + it('falls through to the next candidate base URL when the first is unreachable', async () => { + const fetchMock = vi.fn() + .mockRejectedValueOnce(new Error('connection refused')) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ models: [{ name: 'llama3.2:latest', capabilities: ['completion'], size: 1, details: {} }] }), + }); + global.fetch = fetchMock as unknown as typeof fetch; + + const result = await discoverLocalOllama(['http://127.0.0.1:11434', 'http://localhost:11434']); + expect(result?.baseUrl).toBe('http://localhost:11434'); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('returns null when nothing answers on any candidate', async () => { + global.fetch = vi.fn().mockRejectedValue(new Error('connection refused')) as unknown as typeof fetch; + const result = await discoverLocalOllama(['http://127.0.0.1:11434', 'http://localhost:11434']); + expect(result).toBeNull(); + }); + + it('returns null (not an empty-models result) when Ollama answers with zero models installed', async () => { + global.fetch = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ models: [] }) }) as unknown as typeof fetch; + const result = await discoverLocalOllama(['http://127.0.0.1:11434']); + expect(result).toBeNull(); + }); +}); + +describe('dismissal persistence', () => { + beforeEach(() => { + window.localStorage.clear(); + }); + + it('is not dismissed by default', () => { + expect(isLocalDiscoveryDismissed()).toBe(false); + }); + + it('persists a dismissal across calls', () => { + dismissLocalDiscovery(); + expect(isLocalDiscoveryDismissed()).toBe(true); + }); +}); diff --git a/lib/ai/local-discovery.ts b/lib/ai/local-discovery.ts new file mode 100644 index 00000000..7cdd9268 --- /dev/null +++ b/lib/ai/local-discovery.ts @@ -0,0 +1,112 @@ +// Local-LLM auto-discovery: probes the loopback addresses a local Ollama +// normally binds to, and — if one answers — recommends a model to connect +// with, so a user with Ollama already running never has to type a base URL +// or a model name by hand. +// +// One network call is enough: Ollama's own `/api/tags` already reports +// per-model `capabilities` (completion/embedding/tools/thinking/vision), +// `size`, and `details.parameter_size` — everything the recommendation +// heuristic below needs, with no follow-up `/api/show` round trips. + +export interface DiscoveredLocalModel { + name: string; + capabilities: string[]; + parameterSize: string; + sizeBytes: number; +} + +export interface LocalDiscoveryResult { + baseUrl: string; + models: DiscoveredLocalModel[]; +} + +interface OllamaTagsResponse { + models?: Array<{ + name: string; + capabilities?: string[]; + size?: number; + details?: { parameter_size?: string }; + }>; +} + +// Ollama's own default bind address, plus the hostname form — some setups +// (notably OLLAMA_ORIGINS-restricted CORS allowlists keyed by hostname +// rather than IP) answer one but not the other. +const DEFAULT_PROBE_URLS = ['http://127.0.0.1:11434', 'http://localhost:11434']; +const PROBE_TIMEOUT_MS = 1200; + +async function probeOne(baseUrl: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS); + try { + const res = await fetch(`${baseUrl}/api/tags`, { signal: controller.signal }); + if (!res.ok) return null; + const body = (await res.json()) as OllamaTagsResponse; + const models = (body.models ?? []) + .filter((m) => typeof m.name === 'string' && m.name) + .map((m) => ({ + name: m.name, + capabilities: m.capabilities ?? [], + parameterSize: m.details?.parameter_size ?? '', + sizeBytes: m.size ?? 0, + })); + return models.length > 0 ? { baseUrl, models } : null; + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +/** Tries each candidate in turn (not in parallel — the common case is the + * first one answering, and probing sequentially avoids a burst of + * simultaneous loopback connection attempts for no benefit). */ +export async function discoverLocalOllama( + candidateBaseUrls: readonly string[] = DEFAULT_PROBE_URLS, +): Promise { + for (const baseUrl of candidateBaseUrls) { + const result = await probeOne(baseUrl); + if (result) return result; + } + return null; +} + +/** + * Picks one sensible default out of whatever's installed, so "Connect" + * needs no follow-up decision. Chat-capable models only (never an + * embedding-only model like nomic-embed-text). Among those, prefers + * non-"thinking" models — a reasoning model's chain-of-thought preamble + * reads as a broken first response in a guided setup, however good the + * final answer is — and then the smallest by download size, on the theory + * that the fastest first reply makes the best first impression; a user who + * wants the largest/most capable model for real work can still pick it from + * the full list this only pre-selects. + */ +export function recommendDefaultModel(models: readonly DiscoveredLocalModel[]): string | null { + const chatCapable = models.filter((m) => m.capabilities.includes('completion')); + if (chatCapable.length === 0) return null; + const nonReasoning = chatCapable.filter((m) => !m.capabilities.includes('thinking')); + const pool = nonReasoning.length > 0 ? nonReasoning : chatCapable; + return [...pool].sort((a, b) => a.sizeBytes - b.sizeBytes)[0].name; +} + +/** The largest chat-capable model, for the "most capable" callout next to + * the speed-optimized recommendation above — skipped in the UI when it's + * the same model `recommendDefaultModel` already picked. */ +export function largestModel(models: readonly DiscoveredLocalModel[]): string | null { + const chatCapable = models.filter((m) => m.capabilities.includes('completion')); + if (chatCapable.length === 0) return null; + return [...chatCapable].sort((a, b) => b.sizeBytes - a.sizeBytes)[0].name; +} + +const DISMISSED_KEY = 'vncmail:ai:local-discovery-dismissed'; + +export function isLocalDiscoveryDismissed(): boolean { + if (typeof window === 'undefined') return true; + return window.localStorage.getItem(DISMISSED_KEY) === 'true'; +} + +export function dismissLocalDiscovery(): void { + if (typeof window === 'undefined') return; + window.localStorage.setItem(DISMISSED_KEY, 'true'); +} diff --git a/lib/mail-index/__tests__/store.test.ts b/lib/mail-index/__tests__/store.test.ts index 3c935c32..69a212ff 100644 --- a/lib/mail-index/__tests__/store.test.ts +++ b/lib/mail-index/__tests__/store.test.ts @@ -6,7 +6,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { isSqlcipherAvailable } from '../binding'; import type { SqlcipherConstructor, SqlcipherDatabase, SqlcipherStatement } from '../binding'; import { accountFileToken, getStoreDir, indexDbPath, STORE_DIR_ENV } from '../paths'; -import { MailIndex, openKeyed, toFtsMatchQuery, type IndexDoc } from '../store'; +import { MailIndex, openKeyed, toFtsMatchQuery, toFtsMatchQueryAny, type IndexDoc } from '../store'; describe('toFtsMatchQuery', () => { it('quotes every token so FTS5 operators in user input cannot break the query', () => { @@ -48,6 +48,62 @@ describe('toFtsMatchQuery', () => { }); }); +describe('toFtsMatchQueryAny', () => { + it('drops English function words and OR-joins what is left - the confirmed-live failure this fixes', () => { + // AND-every-token (toFtsMatchQuery) returns 0 hits for this exact + // question against a document that only contains "Villa sul Lago" and + // "check-in" - see app/api/offline/search/route.ts's comment and the + // e2e electron-ai-local-index.spec.ts run that first caught this. + const result = toFtsMatchQueryAny('When is check-in for the Villa sul Lago booking, and what time?'); + expect(result).not.toBeNull(); + expect(result).not.toContain(' AND '); + expect(result).toContain('"check-in"'); + expect(result).toContain('"Villa"'); + expect(result).toContain('"sul"'); + expect(result).toContain('"Lago"'); + expect(result).toContain('"booking"'); + // "time" is the last surviving content word, so it gets the + // prefix-match star - not "Lago", which is merely the last one this + // test happens to name first. + expect(result).toContain('"time"*'); + // Pure stop words, correctly dropped rather than OR-joined as noise that + // would otherwise match almost every document in a mailbox. + expect(result).not.toMatch(/"When"|"is"|"for"|"the"|"and"|"what"/i); + }); + + it('falls back to the unfiltered text when every word is a stop word, rather than searching for nothing', () => { + // "What is this" is 100% stop words - dropping all of them would leave + // zero tokens (a null match, meaning "return everything" is wrong for a + // question shaped like this); falling back to the original text at + // least keeps a real, if weak, query. + const result = toFtsMatchQueryAny('What is this'); + expect(result).not.toBeNull(); + }); + + it('still safely quotes FTS5 syntax characters even after stop-word filtering removes the surrounding noise', () => { + // "OR"/"NEAR" themselves are common enough as English words that this + // builder's stop-word list intentionally drops bare "or" (unlike + // toFtsMatchQuery, which preserves it verbatim - see that test's own + // comment on why: different concern, different guarantee). The safety + // property that DOES still apply here is the one that matters for a + // 500: whatever tokens survive filtering are always quoted before + // reaching FTS5, so a stray `"`/`*`/`(` in real question text can never + // raise a syntax error. + const result = toFtsMatchQueryAny('a" NEAR(bar) baz*'); + expect(result).not.toBeNull(); + expect(result).toContain('"NEAR"'); + expect(result).toContain('"bar"'); + expect(result).toContain('"baz"'); + expect(result).not.toMatch(/fts5|syntax/i); + }); + + it('returns null for input with no usable tokens', () => { + expect(toFtsMatchQueryAny('')).toBeNull(); + expect(toFtsMatchQueryAny('***')).toBeNull(); + expect(toFtsMatchQueryAny(undefined as unknown as string)).toBeNull(); + }); +}); + describe('paths', () => { const original = process.env[STORE_DIR_ENV]; afterEach(() => { diff --git a/lib/mail-index/store.ts b/lib/mail-index/store.ts index ddcdd0da..dddbb3a1 100644 --- a/lib/mail-index/store.ts +++ b/lib/mail-index/store.ts @@ -331,8 +331,12 @@ export class MailIndex { types?: readonly ContentType[]; limit?: number; snippetTokens?: number; + /** 'and' (default): every token required - a deliberate search-box query. + * 'any': stop words dropped, remaining tokens OR-joined, ranked by bm25 - + * a natural-language question (see toFtsMatchQueryAny's docstring). */ + mode?: 'and' | 'any'; }): SearchHit[] { - const match = toFtsMatchQuery(opts.query); + const match = opts.mode === 'any' ? toFtsMatchQueryAny(opts.query) : toFtsMatchQuery(opts.query); if (!match) return []; const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200); @@ -446,34 +450,85 @@ function safeParseObject(v: unknown): Record { } /** - * Turns arbitrary user text into a safe FTS5 MATCH expression. - * - * FTS5's query syntax is not SQL, so parameter binding does NOT protect it: a - * bare `"` or a stray `*`/`NEAR`/`:` in user input raises - * `fts5: syntax error`, which would turn a normal search box into a 500. Every - * token is quoted (making it a literal phrase) and a trailing `*` is added to - * the last token so typing continues to match as the user types. + * Splits and safely quotes raw text into FTS5-safe tokens, shared by both + * query-builders below. Split on anything that isn't a word character or an + * intra-word mark - keeps unicode letters (so "Müller" and "東京" survive) + * via the u flag. Every token is quoted (making it a literal phrase) so a + * bare `"` or a stray `*`/`NEAR`/`:` in user input can never raise FTS5's own + * `fts5: syntax error` - that would turn a normal search into a 500. + */ +function quoteFtsTokens(raw: string): string[] { + return raw + .normalize('NFC') + .split(/[^\p{L}\p{N}_@.'-]+/u) + .map((t) => t.replace(/^['-]+|['-]+$/g, '')) + .filter((t) => t.length > 0) + .slice(0, 24) + .map((t, i, all) => { + const quoted = `"${t.replace(/"/g, '""')}"`; + // Prefix-match only the final token, and only if it's long enough to not + // match half the mailbox. + return i === all.length - 1 && t.length >= 3 ? `${quoted}*` : quoted; + }); +} + +/** + * Turns arbitrary user text into a safe FTS5 MATCH expression, every token + * required (AND-joined). Right for a deliberate, short search-box query, + * where requiring every word is what makes results precise as you type. * * Exported for unit testing - it is the one piece of this file with no * database dependency and the most ways to be wrong. */ export function toFtsMatchQuery(raw: string): string | null { if (typeof raw !== 'string') return null; - // Split on anything that isn't a word character or an intra-word mark. Keeps - // unicode letters (so "Müller" and "東京" survive) via the u flag. - const tokens = raw - .normalize('NFC') - .split(/[^\p{L}\p{N}_@.'-]+/u) - .map((t) => t.replace(/^['-]+|['-]+$/g, '')) - .filter((t) => t.length > 0) - .slice(0, 24); + const tokens = quoteFtsTokens(raw); if (tokens.length === 0) return null; - return tokens - .map((t, i) => { - const quoted = `"${t.replace(/"/g, '""')}"`; - // Prefix-match only the final token, and only if it's long enough to not - // match half the mailbox. - return i === tokens.length - 1 && t.length >= 3 ? `${quoted}*` : quoted; - }) - .join(' AND '); + return tokens.join(' AND '); +} + +// A minimal, well-known set of English function words that carry no +// retrieval signal - kept out of toFtsMatchQueryAny's OR expression so they +// don't drown out the bm25 ranking's actual signal (see below). Deliberately +// NOT applied inside quoteFtsTokens/toFtsMatchQuery: that function's own +// tests rely on "AND"/"OR"/"NOT" surviving verbatim as literal search terms +// (FTS5-keyword-injection safety) - a different concern from this one's job +// of turning a natural-language QUESTION into a good search. +const RETRIEVAL_STOP_WORDS = new Set([ + 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'am', + 'and', 'or', 'but', 'if', 'then', 'than', 'so', 'because', + 'for', 'of', 'to', 'in', 'on', 'at', 'by', 'with', 'from', 'as', 'about', 'into', 'over', 'after', 'before', + 'that', 'this', 'these', 'those', 'what', 'when', 'where', 'who', 'whom', 'which', 'why', 'how', + 'do', 'does', 'did', 'doing', 'done', + 'can', 'could', 'will', 'would', 'shall', 'should', 'may', 'might', 'must', + 'i', 'you', 'he', 'she', 'it', 'we', 'they', 'my', 'your', 'his', 'her', 'its', 'our', 'their', 'me', 'him', 'us', 'them', + 'not', 'no', +]); + +/** + * Turns a natural-language QUESTION into a lenient FTS5 MATCH expression: + * stop words dropped, remaining tokens OR-joined so bm25 ranks by how many + * content words matched instead of requiring every one of them present. + * + * toFtsMatchQuery's strict AND is wrong for this shape of input: a real + * question like "When is check-in for the Villa sul Lago booking?" shares + * almost none of its own function words ("when"/"is"/"for"/"the") with the + * document that actually answers it, so ANDing every token together returns + * nothing - confirmed live: 0 hits for the full question, 2 correct hits for + * the same index once reduced to "Villa sul Lago check-in". The one real + * caller of `/api/offline/search?q=...` is exactly this AI-question shape + * (see that route's own header - no manual search-box UI hits it today), so + * this is the query builder that route now uses, not toFtsMatchQuery. + */ +export function toFtsMatchQueryAny(raw: string): string | null { + if (typeof raw !== 'string') return null; + const withoutStopWords = raw + .split(/\s+/) + .filter((w) => w.length > 0 && !RETRIEVAL_STOP_WORDS.has(w.toLowerCase().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ''))) + .join(' '); + // Every word was a stop word (e.g. "What is this?") - fall back to the + // original text rather than searching for literally nothing. + const tokens = quoteFtsTokens(withoutStopWords.length > 0 ? withoutStopWords : raw); + if (tokens.length === 0) return null; + return tokens.join(' OR '); } diff --git a/playwright.electron-ai.config.ts b/playwright.electron-ai.config.ts new file mode 100644 index 00000000..42d5d791 --- /dev/null +++ b/playwright.electron-ai.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from '@playwright/test'; + +// Separate from playwright.electron.config.ts (which hardcodes testMatch to +// electron-smoke.spec.ts) purely so this one test can get a longer timeout — +// real Ollama inference plus a real multi-round offline sync/reindex chain +// legitimately takes longer than the smoke suite's 60s budget. +export default defineConfig({ + testDir: './e2e', + testMatch: 'electron-ai-local-index.spec.ts', + timeout: 120000, + retries: 0, + use: { + trace: 'retain-on-failure', + }, + workers: 1, +});