promote: local-LLM auto-discovery + real local-index/local-LLM bugfix + e2e coverage (dev→main)
This commit is contained in:
+40
-106
@@ -1,80 +1,30 @@
|
|||||||
# GitLab-CI dev→prod pipeline for VNCmail+ — GitOps via ArgoCD.
|
# GitLab-CI dev→prod pipeline for VNCmail+ — GitOps via ArgoCD.
|
||||||
#
|
#
|
||||||
# Revised after direct inspection of the real infrastructure found ArgoCD
|
|
||||||
# already installed (idle, zero Applications) on the dev-k8s-1/2/3 cluster.
|
|
||||||
# That's more idiomatic than a runner-executes-kubectl design, and it means
|
|
||||||
# this pipeline needs ZERO cluster credentials — CI only ever talks to the
|
|
||||||
# container registry and to this git repo. ArgoCD (which already has
|
|
||||||
# whatever cluster access it needs, set up once when its Applications were
|
|
||||||
# registered — see deploy/argocd/) is what actually applies anything.
|
|
||||||
#
|
|
||||||
# Design:
|
# Design:
|
||||||
# - One image name, environment lives only in the tag. No more -dev/-beta
|
|
||||||
# name confusion from the old GitHub Actions workflow.
|
|
||||||
# - MR into `dev`: verify only (typecheck/lint/unit test/build check). No
|
# - MR into `dev`: verify only (typecheck/lint/unit test/build check). No
|
||||||
# push, no deploy — this is the multi-developer merge gate.
|
# push, no deploy — this is the multi-developer merge gate.
|
||||||
# - Push to `dev`: build+push an immutable `sha-<sha>` tag, then commit a
|
# - Push to `dev`: build+push an immutable `sha-<sha>` tag with Docker +
|
||||||
# one-line tag-bump into overlays/dev/image-tag/kustomization.yaml
|
# docker-in-docker, then commit a one-line tag-bump into
|
||||||
# (`[skip ci]`, so this doesn't retrigger itself). ArgoCD's `vncmail-dev`
|
# overlays/dev/image-tag/kustomization.yaml (`[skip ci]`). ArgoCD's
|
||||||
# Application has automated sync — it notices the git change and applies
|
# `vncmail-dev` Application syncs it automatically.
|
||||||
# it. No approval needed, dev always deploys, and this job never touches
|
# - Push to `main`: NEVER rebuilds. `main` only advances via
|
||||||
# the cluster directly.
|
|
||||||
# - Push to `main`: NEVER rebuilds. `main` only ever advances via
|
|
||||||
# `git merge --ff-only dev`, so main's HEAD commit already has a built
|
# `git merge --ff-only dev`, so main's HEAD commit already has a built
|
||||||
# image (the same sha- tag dev already deployed). This job just bumps
|
# image. This job just bumps overlays/prod/image-tag/kustomization.yaml
|
||||||
# overlays/prod/image-tag/kustomization.yaml to point at that same tag.
|
# to point at that same tag. The actual promotion gate is a HUMAN
|
||||||
# The actual promotion gate is a HUMAN clicking Sync on the `vncmail-prod` ArgoCD
|
# clicking Sync on the `vncmail-prod` ArgoCD Application.
|
||||||
# Application (deliberately NOT automated sync) — not a GitLab manual
|
|
||||||
# job, since ArgoCD already provides that exact gate more directly.
|
|
||||||
# Until prod Stalwart/hostname/secrets are real (see VNCMAIL-SETUP.md),
|
|
||||||
# nobody should click that Sync button — but nothing here does it for
|
|
||||||
# you either way.
|
|
||||||
#
|
#
|
||||||
# Deliberately single-platform (linux/amd64) — this pipeline serves two
|
# Deliberately single-platform (linux/amd64) — this pipeline serves two
|
||||||
# known amd64 microk8s clusters, not public multi-arch distribution (that's
|
# known amd64 microk8s clusters, not public multi-arch distribution (that's
|
||||||
# what the GHCR release workflows are for, untouched by this file).
|
# what the GHCR release workflows are for, untouched by this file).
|
||||||
#
|
#
|
||||||
# Registry history (so nobody re-litigates this from scratch). GitLab's own
|
# Prerequisite this file assumes:
|
||||||
# Container Registry was tried twice and does not work on this server:
|
# - A GitLab Runner with Docker-in-Docker service support (Kubernetes or
|
||||||
#
|
# Docker executor). The `docker:28.4.0-dind` service requires privileged
|
||||||
# Round 1: registry_external_url was unset, so $CI_REGISTRY was empty and
|
# mode on most Kubernetes executors.
|
||||||
# docker login silently fell through to Docker Hub.
|
|
||||||
# Round 2: after the server-side config, the project's sidebar DID show
|
|
||||||
# "Container Registry" and docker login DID succeed - but the push
|
|
||||||
# failed 403. Diagnosed 2026-08-05 by curling the vhost directly:
|
|
||||||
#
|
|
||||||
# $ curl -i https://registry.gitlab.vnc.biz/v2/
|
|
||||||
# www-authenticate: Bearer realm="http://gitlab.vnc.biz/jwt/auth",
|
|
||||||
# service="dependency_proxy"
|
|
||||||
# x-runtime: 0.020470
|
|
||||||
# x-gitlab-meta: {"correlation_id":...}
|
|
||||||
#
|
|
||||||
# x-runtime/x-gitlab-meta are RAILS headers, and the service is
|
|
||||||
# "dependency_proxy" - so nginx routes that hostname to the GitLab
|
|
||||||
# Rails app, which reads /v2/ as the dependency proxy (a Docker Hub
|
|
||||||
# pull-through cache), NOT to the registry container. The registry
|
|
||||||
# service was never actually wired behind that vhost. That is why
|
|
||||||
# login worked (Rails issues an unscoped dependency-proxy token)
|
|
||||||
# while a scoped :push request 403'd - the dependency proxy has no
|
|
||||||
# push concept at all.
|
|
||||||
#
|
|
||||||
# Fixing that is an nginx/omnibus change on the GitLab server (registry
|
|
||||||
# service must actually listen behind registry.gitlab.vnc.biz), not something
|
|
||||||
# any .gitlab-ci.yml can reach. Until someone does that, GHCR it is - the
|
|
||||||
# same image the sandbox already pulls, and confirmed public so the cluster
|
|
||||||
# needs no imagePullSecrets (see deploy/k8s/base/deployment.yaml).
|
|
||||||
#
|
|
||||||
# Prerequisite this file assumes (documented in VNCMAIL-SETUP.md, not
|
|
||||||
# something this file can set up itself):
|
|
||||||
# - A GitHub PAT with `write:packages` for ghcr.io/brvncde-dotcom as
|
|
||||||
# $GITLAB_CI_GHCR_TOKEN, plus the matching GitHub username as
|
|
||||||
# $GITLAB_CI_GHCR_USER — both masked+protected CI/CD variables. A GitHub
|
|
||||||
# credential can only come from GitHub; nothing GitLab-native substitutes.
|
|
||||||
# - A GitLab Runner (any kind — no cluster access needed at all now).
|
|
||||||
# - Either "allow this job token to push to this project" enabled
|
# - Either "allow this job token to push to this project" enabled
|
||||||
# (Settings → CI/CD → Job token permissions), OR a project access token
|
# (Settings → CI/CD → Job token permissions), OR a project access token
|
||||||
# with `write_repository` scope in $GITLAB_PUSH_TOKEN. The job below
|
# with `write_repository` scope in $GITLAB_PUSH_TOKEN. The bump jobs
|
||||||
# tries CI_JOB_TOKEN first (see the script).
|
# try CI_JOB_TOKEN first (see the script).
|
||||||
#
|
#
|
||||||
# deploy/k8s/ca/ (the EJBCA internal CA) is never referenced anywhere below,
|
# deploy/k8s/ca/ (the EJBCA internal CA) is never referenced anywhere below,
|
||||||
# and neither ArgoCD Application in deploy/argocd/ points at it — that stays
|
# and neither ArgoCD Application in deploy/argocd/ points at it — that stays
|
||||||
@@ -87,11 +37,14 @@ stages:
|
|||||||
- bump-prod
|
- bump-prod
|
||||||
|
|
||||||
variables:
|
variables:
|
||||||
# GHCR, not GitLab's own registry - see the "Registry history" note in the
|
IMAGE: $CI_REGISTRY_IMAGE
|
||||||
# header for the curl output proving why. Same image the sandbox already
|
|
||||||
# pulls today.
|
|
||||||
IMAGE: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
|
||||||
GIT_STRATEGY: clone
|
GIT_STRATEGY: clone
|
||||||
|
DOCKER_DRIVER: overlay2
|
||||||
|
# DinD service is reached at the `docker` alias (set explicitly on the
|
||||||
|
# service below), not localhost. TLS disabled so the daemon listens on
|
||||||
|
# plaintext 2375 — same pattern as the working vnc-localidp pipeline.
|
||||||
|
DOCKER_HOST: tcp://docker:2375
|
||||||
|
DOCKER_TLS_CERTDIR: ""
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# verify — required check on every MR into dev. No registry, no cluster.
|
# verify — required check on every MR into dev. No registry, no cluster.
|
||||||
@@ -118,45 +71,24 @@ verify:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
build:
|
build:
|
||||||
stage: build
|
stage: build
|
||||||
# Kaniko builds OCI images without a Docker daemon, so it needs neither a
|
image: docker:28.4.0
|
||||||
# dind service nor a privileged pod — GitLab's own recommended approach
|
services:
|
||||||
# for the Kubernetes executor specifically. The docker:27-cli + dind
|
- name: docker:28.4.0-dind
|
||||||
# combination that was here before this got as far as a successful
|
alias: docker
|
||||||
# $CI_REGISTRY login, then failed every way it was pointed
|
|
||||||
# (unix:///var/run/docker.sock, tcp://docker:2375, tcp://localhost:2375):
|
|
||||||
# the dind container itself was never actually listening, which on this
|
|
||||||
# executor means it needs `privileged: true` in the runner's own
|
|
||||||
# config.toml — a cluster/GitLab-admin setting outside this file's
|
|
||||||
# control. Kaniko sidesteps that requirement entirely rather than chasing
|
|
||||||
# runner permissions further, and is also the safer default on a shared
|
|
||||||
# cluster (no privileged containers at all).
|
|
||||||
image:
|
|
||||||
name: gcr.io/kaniko-project/executor:v1.23.2-debug
|
|
||||||
entrypoint: [""]
|
|
||||||
rules:
|
rules:
|
||||||
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
|
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
|
||||||
|
before_script:
|
||||||
|
- until docker info; do sleep 1; done
|
||||||
|
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
|
||||||
script:
|
script:
|
||||||
# Fail loudly and immediately if the GHCR credentials aren't configured,
|
|
||||||
# rather than letting kaniko get all the way through a full Next.js build
|
|
||||||
# and only then 403 on push (which is exactly how the GitLab-registry
|
|
||||||
# attempt burned several pipeline runs).
|
|
||||||
- |
|
|
||||||
if [ -z "$GITLAB_CI_GHCR_TOKEN" ] || [ -z "$GITLAB_CI_GHCR_USER" ]; then
|
|
||||||
echo "ERROR: \$GITLAB_CI_GHCR_TOKEN and/or \$GITLAB_CI_GHCR_USER are not set."
|
|
||||||
echo "Add both under Settings -> CI/CD -> Variables (masked + protected)."
|
|
||||||
echo "The token is a GitHub PAT with the write:packages scope."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
- mkdir -p /kaniko/.docker
|
|
||||||
- |
|
|
||||||
echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"$(printf '%s:%s' "$GITLAB_CI_GHCR_USER" "$GITLAB_CI_GHCR_TOKEN" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
|
|
||||||
- >
|
- >
|
||||||
/kaniko/executor
|
docker build
|
||||||
--context "$CI_PROJECT_DIR"
|
|
||||||
--dockerfile "$CI_PROJECT_DIR/Dockerfile"
|
|
||||||
--build-arg GIT_COMMIT=$CI_COMMIT_SHA
|
--build-arg GIT_COMMIT=$CI_COMMIT_SHA
|
||||||
--destination "$IMAGE:sha-$CI_COMMIT_SHORT_SHA"
|
-t "$IMAGE:sha-$CI_COMMIT_SHORT_SHA"
|
||||||
--destination "$IMAGE:dev-latest"
|
-t "$IMAGE:dev-latest"
|
||||||
|
.
|
||||||
|
- docker push "$IMAGE:sha-$CI_COMMIT_SHORT_SHA"
|
||||||
|
- docker push "$IMAGE:dev-latest"
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# bump-dev — no cluster access. Commits the just-built tag into the overlay
|
# bump-dev — no cluster access. Commits the just-built tag into the overlay
|
||||||
@@ -164,7 +96,9 @@ build:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
bump-dev:
|
bump-dev:
|
||||||
stage: bump-dev
|
stage: bump-dev
|
||||||
image: alpine/git:2.47.0
|
# alpine/git:2.47.0 was never published on Docker Hub — the 2.47.x line
|
||||||
|
# starts at 2.47.1. Using 2.47.2 (latest 2.47.x).
|
||||||
|
image: alpine/git:2.47.2
|
||||||
rules:
|
rules:
|
||||||
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
|
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "dev"'
|
||||||
script:
|
script:
|
||||||
@@ -176,7 +110,7 @@ bump-dev:
|
|||||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||||
kind: Component
|
kind: Component
|
||||||
images:
|
images:
|
||||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
- name: vncmail-plus
|
||||||
newName: $IMAGE
|
newName: $IMAGE
|
||||||
newTag: $TAG
|
newTag: $TAG
|
||||||
EOF
|
EOF
|
||||||
@@ -199,7 +133,7 @@ bump-dev:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
bump-prod:
|
bump-prod:
|
||||||
stage: bump-prod
|
stage: bump-prod
|
||||||
image: alpine/git:2.47.0
|
image: alpine/git:2.47.2
|
||||||
rules:
|
rules:
|
||||||
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"'
|
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"'
|
||||||
script:
|
script:
|
||||||
@@ -215,7 +149,7 @@ bump-prod:
|
|||||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||||
kind: Component
|
kind: Component
|
||||||
images:
|
images:
|
||||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
- name: vncmail-plus
|
||||||
newName: $IMAGE
|
newName: $IMAGE
|
||||||
newTag: $TAG
|
newTag: $TAG
|
||||||
EOF
|
EOF
|
||||||
|
|||||||
@@ -79,7 +79,12 @@ export async function GET(request: NextRequest) {
|
|||||||
try {
|
try {
|
||||||
const stats = index.stats();
|
const stats = index.stats();
|
||||||
if (!query) return { hits: [] as SearchHit[], stats };
|
if (!query) return { hits: [] as SearchHit[], stats };
|
||||||
return { hits: index.search({ query, types, limit }), stats: wantStats ? stats : undefined };
|
// 'any': this route is the AI/RAG retrieval surface (see module
|
||||||
|
// header) - its one real caller sends natural-language questions,
|
||||||
|
// not deliberate search-box keywords, so strict AND-every-token
|
||||||
|
// matching (the default) drops nearly all of them. See
|
||||||
|
// toFtsMatchQueryAny's docstring for the confirmed-live failure.
|
||||||
|
return { hits: index.search({ query, types, limit, mode: 'any' }), stats: wantStats ? stats : undefined };
|
||||||
} finally {
|
} finally {
|
||||||
index.close();
|
index.close();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||||
import { RefreshCw, CheckCircle, AlertTriangle, Loader2, Plus, Trash2 } from 'lucide-react';
|
import { RefreshCw, CheckCircle, AlertTriangle, Loader2, Plus, Trash2, Sparkles, X } from 'lucide-react';
|
||||||
import { SettingsSection, SettingItem, ToggleSwitch, RadioGroup, Select } from './settings-section';
|
import { SettingsSection, SettingItem, ToggleSwitch, RadioGroup, Select } from './settings-section';
|
||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
import { apiFetch } from '@/lib/browser-navigation';
|
import { apiFetch } from '@/lib/browser-navigation';
|
||||||
@@ -9,6 +9,14 @@ import { DEFAULT_AI_POLICY, type AiPolicy } from '@/lib/ai/types';
|
|||||||
import { supportsLocalLlm, localLlmNeedsCorsSetup } from '@/lib/platform-capabilities';
|
import { supportsLocalLlm, localLlmNeedsCorsSetup } from '@/lib/platform-capabilities';
|
||||||
import { getAiApiKey, setAiApiKey, clearAiApiKey } from '@/lib/ai/key-store';
|
import { getAiApiKey, setAiApiKey, clearAiApiKey } from '@/lib/ai/key-store';
|
||||||
import { loadAiSettings, saveAiSettings, createProfile, type AiLocalSettings } from '@/lib/ai/local-settings';
|
import { loadAiSettings, saveAiSettings, createProfile, type AiLocalSettings } from '@/lib/ai/local-settings';
|
||||||
|
import {
|
||||||
|
discoverLocalOllama,
|
||||||
|
recommendDefaultModel,
|
||||||
|
largestModel,
|
||||||
|
isLocalDiscoveryDismissed,
|
||||||
|
dismissLocalDiscovery,
|
||||||
|
type LocalDiscoveryResult,
|
||||||
|
} from '@/lib/ai/local-discovery';
|
||||||
import {
|
import {
|
||||||
askMail,
|
askMail,
|
||||||
listLocalModels,
|
listLocalModels,
|
||||||
@@ -91,6 +99,47 @@ export function AiAssistantSettings() {
|
|||||||
}
|
}
|
||||||
}, [settings.localBaseUrl]);
|
}, [settings.localBaseUrl]);
|
||||||
|
|
||||||
|
// ── Local discovery — proactively find an already-running Ollama and
|
||||||
|
// offer a one-click connect, rather than making the user hunt down and
|
||||||
|
// type a base URL + model name by hand. ──
|
||||||
|
const [discovery, setDiscovery] = useState<LocalDiscoveryResult | null>(null);
|
||||||
|
const [discoveryDismissed, setDiscoveryDismissed] = useState(true);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setDiscoveryDismissed(isLocalDiscoveryDismissed());
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!canUseLocal || discoveryDismissed || settings.localModel) return;
|
||||||
|
let cancelled = false;
|
||||||
|
(async () => {
|
||||||
|
const result = await discoverLocalOllama();
|
||||||
|
if (!cancelled) setDiscovery(result);
|
||||||
|
})();
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [canUseLocal, discoveryDismissed, settings.localModel]);
|
||||||
|
|
||||||
|
const connectDiscoveredLocal = useCallback(() => {
|
||||||
|
if (!discovery) return;
|
||||||
|
const recommended = recommendDefaultModel(discovery.models) ?? discovery.models[0]?.name ?? null;
|
||||||
|
if (!recommended) return;
|
||||||
|
setSettings((prev) => {
|
||||||
|
const next: AiLocalSettings = { ...prev, provider: 'local', localBaseUrl: discovery.baseUrl, localModel: recommended };
|
||||||
|
saveAiSettings(next);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
setLocalModels(discovery.models.filter((m) => m.capabilities.includes('completion')).map((m) => m.name));
|
||||||
|
setDiscovery(null);
|
||||||
|
}, [discovery]);
|
||||||
|
|
||||||
|
const dismissDiscoveryBanner = useCallback(() => {
|
||||||
|
dismissLocalDiscovery();
|
||||||
|
setDiscoveryDismissed(true);
|
||||||
|
setDiscovery(null);
|
||||||
|
}, []);
|
||||||
|
|
||||||
// ── Server provider ──
|
// ── Server provider ──
|
||||||
const [serverModels, setServerModels] = useState<string[]>([]);
|
const [serverModels, setServerModels] = useState<string[]>([]);
|
||||||
const [refreshingServer, setRefreshingServer] = useState(false);
|
const [refreshingServer, setRefreshingServer] = useState(false);
|
||||||
@@ -209,8 +258,35 @@ export function AiAssistantSettings() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const discoveryRecommended = discovery ? recommendDefaultModel(discovery.models) : null;
|
||||||
|
const discoveryLargest = discovery ? largestModel(discovery.models) : null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
|
{discovery && discoveryRecommended && (
|
||||||
|
<div className="flex items-start gap-3 rounded-lg border border-primary/30 bg-primary/5 p-4">
|
||||||
|
<Sparkles className="w-5 h-5 mt-0.5 text-primary shrink-0" />
|
||||||
|
<div className="flex-1 min-w-0 space-y-2">
|
||||||
|
<p className="text-sm font-medium text-foreground">Local AI found on this machine</p>
|
||||||
|
<p className="text-sm text-muted-foreground">
|
||||||
|
Ollama is running at {discovery.baseUrl} with {discovery.models.length} model{discovery.models.length === 1 ? '' : 's'} installed.
|
||||||
|
Recommended for quick answers: <span className="font-medium text-foreground">{discoveryRecommended}</span>.
|
||||||
|
{discoveryLargest && discoveryLargest !== discoveryRecommended && (
|
||||||
|
<> Also available for higher-quality answers: <span className="font-medium text-foreground">{discoveryLargest}</span>.</>
|
||||||
|
)}
|
||||||
|
</p>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button size="sm" onClick={connectDiscoveredLocal}>
|
||||||
|
<Sparkles className="w-3.5 h-3.5 me-1.5" /> Connect
|
||||||
|
</Button>
|
||||||
|
<Button size="sm" variant="outline" onClick={dismissDiscoveryBanner}>
|
||||||
|
<X className="w-3.5 h-3.5 me-1.5" /> Not now
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<SettingsSection
|
<SettingsSection
|
||||||
title="AI Assistant"
|
title="AI Assistant"
|
||||||
description="Ask questions about your synced mail. Local runs entirely on this machine's own model runtime; server is centrally hosted and licensed per seat; public sends your question to a provider you choose, using your own API key."
|
description="Ask questions about your synced mail. Local runs entirely on this machine's own model runtime; server is centrally hosted and licensed per seat; public sends your question to a provider you choose, using your own API key."
|
||||||
|
|||||||
+14
-15
@@ -59,10 +59,9 @@ is CI's job to decide — it's an explicit, human-triggered event.
|
|||||||
| 7 | Ingress `vncmail-plus` | `base/ingress.yaml` (+ overlay patches for prod) | TLS host |
|
| 7 | Ingress `vncmail-plus` | `base/ingress.yaml` (+ overlay patches for prod) | TLS host |
|
||||||
|
|
||||||
**Image:** CI builds and pushes to `registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus`
|
**Image:** CI builds and pushes to `registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus`
|
||||||
(tag `sha-<sha>` per deploy, moving pointers `dev-latest`/`prod-latest`). The
|
(tag `sha-<sha>` per deploy, moving pointer `dev-latest`). The generic `vncmail-plus`
|
||||||
`ghcr.io/brvncde-dotcom/vncmail-plus-dev` image referenced in `base/deployment.yaml`
|
image name in `base/deployment.yaml` is a placeholder — kustomize's image-tag
|
||||||
is a legacy default only — CI overrides it per-deploy via `kubectl set image`,
|
Component replaces it with the real registry path on every deploy.
|
||||||
so what's committed there never needs to track what's actually running.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -98,15 +97,15 @@ pick up the fix):
|
|||||||
```bash
|
```bash
|
||||||
cd deploy/k8s/overlays/dev # or overlays/prod, once real
|
cd deploy/k8s/overlays/dev # or overlays/prod, once real
|
||||||
|
|
||||||
# a) Image-pull secret — the registry package is private.
|
# a) Image-pull secret — the GitLab registry requires authentication.
|
||||||
kubectl create secret docker-registry ghcr-pull \
|
# Use a project deploy token with `read_registry` scope, or the CI job
|
||||||
|
# token (short-lived — better for CI, not for long-running clusters).
|
||||||
|
kubectl create secret docker-registry gitlab-registry \
|
||||||
--namespace vncmail \
|
--namespace vncmail \
|
||||||
--docker-server=ghcr.io \
|
--docker-server=registry.gitlab.vnc.biz \
|
||||||
--docker-username=brvncde-dotcom \
|
--docker-username=<deploy-token-name> \
|
||||||
--docker-password='<GITHUB_PAT_read:packages>' \
|
--docker-password='<deploy-token-secret>' \
|
||||||
--docker-email=br@vnc.biz
|
--docker-email=ci@vnc.biz
|
||||||
# Once CI has cut over to registry.gitlab.vnc.biz, this becomes a
|
|
||||||
# docker-registry secret for that registry instead — see VNCMAIL-SETUP.md.
|
|
||||||
|
|
||||||
# b) App config secret — copy the template, set a real SESSION_SECRET, apply.
|
# b) App config secret — copy the template, set a real SESSION_SECRET, apply.
|
||||||
cp secret.example.yaml secret.yaml
|
cp secret.example.yaml secret.yaml
|
||||||
@@ -117,8 +116,8 @@ kubectl apply -f secret.yaml
|
|||||||
kubectl apply -k .
|
kubectl apply -k .
|
||||||
```
|
```
|
||||||
|
|
||||||
> Alternative to (a): make the registry package public, then delete the
|
> Alternative to (a): make the GitLab container registry public for this
|
||||||
> `imagePullSecrets:` block from `base/deployment.yaml`.
|
> project, then delete the `imagePullSecrets:` block from `base/deployment.yaml`.
|
||||||
|
|
||||||
After this one-time setup, routine deploys to `dev` happen automatically via
|
After this one-time setup, routine deploys to `dev` happen automatically via
|
||||||
CI on every push — see "Routine deploys go through CI now" above. This
|
CI on every push — see "Routine deploys go through CI now" above. This
|
||||||
@@ -170,7 +169,7 @@ ArgoCD re-sync.
|
|||||||
|
|
||||||
| Symptom | Cause / fix |
|
| Symptom | Cause / fix |
|
||||||
|---------|-------------|
|
|---------|-------------|
|
||||||
| Pod `ImagePullBackOff` | `ghcr-pull` secret missing/expired, or package still private. Recreate the secret (§3a) or make the package public. |
|
| Pod `ImagePullBackOff` | `gitlab-registry` secret missing/expired, or token lacks `read_registry`. Recreate the secret (§3a) or make the registry public. |
|
||||||
| Pod `CrashLoopBackOff`, logs show `EACCES`/permission on `/app/data` | Volume not writable by uid 1001. `securityContext.fsGroup: 1001` is set in `base/deployment.yaml` — keep it; some storage drivers also need it on the PVC. |
|
| Pod `CrashLoopBackOff`, logs show `EACCES`/permission on `/app/data` | Volume not writable by uid 1001. `securityContext.fsGroup: 1001` is set in `base/deployment.yaml` — keep it; some storage drivers also need it on the PVC. |
|
||||||
| PVC stuck `Pending` | Wrong `storageClassName` in `base/pvc.yaml`. Set it to one from `kubectl get sc`. |
|
| PVC stuck `Pending` | Wrong `storageClassName` in `base/pvc.yaml`. Set it to one from `kubectl get sc`. |
|
||||||
| Ingress has no address / no cert | Wrong `ingressClassName` or cert issuer. Match bulwark's (§2). Check `kubectl -n vncmail describe ingress vncmail-plus`. |
|
| Ingress has no address / no cert | Wrong `ingressClassName` or cert issuer. Match bulwark's (§2). Check `kubectl -n vncmail describe ingress vncmail-plus`. |
|
||||||
|
|||||||
@@ -23,20 +23,18 @@ spec:
|
|||||||
fsGroup: 1001
|
fsGroup: 1001
|
||||||
runAsUser: 1001
|
runAsUser: 1001
|
||||||
runAsGroup: 1001
|
runAsGroup: 1001
|
||||||
# Confirmed 2026-08-05: ghcr.io/brvncde-dotcom/vncmail-plus-dev IS public
|
# The GitLab container registry is private by default. Nodes need a
|
||||||
# (anonymous token pull succeeded) — no imagePullSecrets needed. This is
|
# docker-registry secret named `gitlab-registry` in the target namespace.
|
||||||
# deploy/k8s/README.md's own documented alternative to creating a
|
# Create it once per environment during first-time setup
|
||||||
# ghcr-pull secret. Removed rather than left referencing a
|
# (see deploy/k8s/README.md §3a).
|
||||||
# not-yet-created secret, which would otherwise block every pod from
|
imagePullSecrets:
|
||||||
# starting regardless of the image being public (kubelet fails to
|
- name: gitlab-registry
|
||||||
# resolve a missing imagePullSecrets entry before it ever gets to
|
|
||||||
# deciding whether auth was actually required).
|
|
||||||
containers:
|
containers:
|
||||||
- name: vncmail-plus
|
- name: vncmail-plus
|
||||||
# Default/legacy value — CI overrides the image per-deploy via
|
# Generic placeholder — the real image name + tag are injected by the
|
||||||
# `kustomize edit set image`, so what's committed here never goes
|
# image-tag kustomize Component on every deploy (see
|
||||||
# stale. For a one-off manual apply, pin a digest instead of :latest.
|
# overlays/*/image-tag/kustomization.yaml, rewritten by CI).
|
||||||
image: ghcr.io/brvncde-dotcom/vncmail-plus-dev:latest
|
image: vncmail-plus:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 3000
|
- containerPort: 3000
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ metadata:
|
|||||||
name: vncmail-plus
|
name: vncmail-plus
|
||||||
annotations:
|
annotations:
|
||||||
cert-manager.io/cluster-issuer: CHANGEME
|
cert-manager.io/cluster-issuer: CHANGEME
|
||||||
|
traefik.ingress.kubernetes.io/router.middlewares: traefik-redirect-to-https@kubernetescrd
|
||||||
spec:
|
spec:
|
||||||
ingressClassName: traefik
|
ingressClassName: traefik
|
||||||
tls:
|
tls:
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
# Owned by CI (the bump-dev job in .gitlab-ci.yml), not by hand. Kept as its
|
# Owned by CI (bump-dev job in .gitlab-ci.yml) - regenerated every
|
||||||
# own Component so CI only ever rewrites this 6-line file, never the parent
|
# push to dev. Do not hand-edit; edits here get overwritten.
|
||||||
# overlays/dev/kustomization.yaml (structure/patches there stay under normal
|
|
||||||
# code review — CI regenerating a whole hand-maintained file on every push
|
|
||||||
# would silently revert any change made there between deploys).
|
|
||||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||||
kind: Component
|
kind: Component
|
||||||
images:
|
images:
|
||||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
- name: vncmail-plus
|
||||||
newName: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus
|
||||||
newTag: sha-147660a
|
newTag: sha-2a8778c9
|
||||||
|
|||||||
@@ -4,19 +4,6 @@
|
|||||||
# is pure waste - the content behind that tag can never change, so re-pulling
|
# is pure waste - the content behind that tag can never change, so re-pulling
|
||||||
# it on every pod start only adds a registry round-trip and a hard dependency
|
# it on every pod start only adds a registry round-trip and a hard dependency
|
||||||
# on the registry being reachable at scheduling time.
|
# on the registry being reachable at scheduling time.
|
||||||
#
|
|
||||||
# It is also load-bearing right now: until CI can actually push (GitLab's
|
|
||||||
# registry vhost serves Rails, not the registry - see .gitlab-ci.yml's
|
|
||||||
# "Registry history" note), sha- tagged images are side-loaded straight into
|
|
||||||
# each node's containerd:
|
|
||||||
#
|
|
||||||
# docker save --platform linux/amd64 -o vncmail.tar <image>:<tag>
|
|
||||||
# scp vncmail.tar dev-k8s-N:/tmp/ && ssh dev-k8s-N \
|
|
||||||
# 'microk8s ctr images import /tmp/vncmail.tar'
|
|
||||||
#
|
|
||||||
# imported to ALL of dev-k8s-1/2/3 so the pod can schedule anywhere. With
|
|
||||||
# Always, kubelet would ignore that local image and fail on a registry pull
|
|
||||||
# for a tag the registry has never seen.
|
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
|
|||||||
@@ -1,14 +1,12 @@
|
|||||||
# dev-k8s cluster confirmed to have a `letsencrypt-staging` ClusterIssuer
|
# Dev now uses letsencrypt-prod so the sandbox certificate is trusted by
|
||||||
# already (no `letsencrypt-prod` exists there) - staging avoids burning
|
# browsers. Both letsencrypt-staging and letsencrypt-prod ClusterIssuers
|
||||||
# Let's Encrypt's real rate limits while this is still being stood up.
|
# exist on the dev-k8s cluster (see VNCiAC-dev-cluster-runbook.md §7).
|
||||||
# vncmail.sandbox.vnc.de DNS does not point here yet either - this is the
|
|
||||||
# intended host, not a live one (see VNCMAIL-SETUP.md for what's still open).
|
|
||||||
apiVersion: networking.k8s.io/v1
|
apiVersion: networking.k8s.io/v1
|
||||||
kind: Ingress
|
kind: Ingress
|
||||||
metadata:
|
metadata:
|
||||||
name: vncmail-plus
|
name: vncmail-plus
|
||||||
annotations:
|
annotations:
|
||||||
cert-manager.io/cluster-issuer: letsencrypt-staging
|
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||||
spec:
|
spec:
|
||||||
tls:
|
tls:
|
||||||
- hosts:
|
- hosts:
|
||||||
|
|||||||
@@ -7,6 +7,6 @@
|
|||||||
apiVersion: kustomize.config.k8s.io/v1alpha1
|
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||||
kind: Component
|
kind: Component
|
||||||
images:
|
images:
|
||||||
- name: ghcr.io/brvncde-dotcom/vncmail-plus-dev
|
- name: vncmail-plus
|
||||||
newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus
|
newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus
|
||||||
newTag: not-yet-promoted
|
newTag: not-yet-promoted
|
||||||
|
|||||||
@@ -0,0 +1,199 @@
|
|||||||
|
import { test, expect, _electron as electron } from '@playwright/test';
|
||||||
|
import type { ElectronApplication, Page } from '@playwright/test';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Proves the two hardest-to-fake claims about the AI Assistant's `local`
|
||||||
|
* class in the REAL packaged desktop shell, not a browser tab:
|
||||||
|
*
|
||||||
|
* 1. The LLM genuinely runs locally — a direct browser-side fetch to
|
||||||
|
* this machine's own Ollama (127.0.0.1:11434), never proxied through
|
||||||
|
* this app's backend.
|
||||||
|
* 2. It is genuinely grounded in the ENCRYPTED LOCAL SQLITE/FTS5 MAIL
|
||||||
|
* INDEX (lib/mail-index/**), not the separate real-JMAP-embeddings
|
||||||
|
* server leg (lib/ai/retrieval/mail-embeddings.ts) — AI_SERVER_BASE_URL
|
||||||
|
* is deliberately left UNSET here so only the local FTS leg can
|
||||||
|
* supply retrieval context. If this test passes, the local index
|
||||||
|
* leg is the only possible source of the grounded answer.
|
||||||
|
*
|
||||||
|
* Needs a real launch through electron/main.ts's startStandaloneServer(),
|
||||||
|
* not ELECTRON_LOAD_URL — that's the only code path that wires up the
|
||||||
|
* fd-3 key channel / safeStorage the encrypted index depends on (see
|
||||||
|
* integration/tests/12-electron-mail-index.spec.ts's header for the full
|
||||||
|
* reasoning). That function picks a random free port every launch, which
|
||||||
|
* would make it impossible to also point DEV_MOCK_JMAP's JMAP_SERVER_URL
|
||||||
|
* at this same server's own /api/dev-jmap route — hence
|
||||||
|
* VNCMAIL_TEST_FIXED_PORT, a narrow, off-by-default escape hatch added to
|
||||||
|
* electron/main.ts specifically to make this test possible without a real
|
||||||
|
* Stalwart fixture.
|
||||||
|
*
|
||||||
|
* Requires a real Ollama already running on this machine with at least one
|
||||||
|
* completion-capable model installed — skips (not fails) otherwise, since
|
||||||
|
* "no local LLM on this machine" is an environment fact, not a bug.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const projectRoot = path.resolve(__dirname, '..');
|
||||||
|
const FIXED_PORT = 39217;
|
||||||
|
const ORIGIN = `http://127.0.0.1:${FIXED_PORT}`;
|
||||||
|
|
||||||
|
async function ollamaIsUp(): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const res = await fetch('http://127.0.0.1:11434/api/tags');
|
||||||
|
if (!res.ok) return false;
|
||||||
|
const body = (await res.json()) as { models?: Array<{ capabilities?: string[] }> };
|
||||||
|
return (body.models ?? []).some((m) => !m.capabilities || m.capabilities.includes('completion'));
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test.describe('Electron desktop shell - local LLM answers from the real encrypted mail index', () => {
|
||||||
|
let electronApp: ElectronApplication;
|
||||||
|
let appWindow: Page;
|
||||||
|
let userDataDir: string;
|
||||||
|
|
||||||
|
test.beforeAll(async () => {
|
||||||
|
if (!(await ollamaIsUp())) {
|
||||||
|
test.skip(true, 'No local Ollama with a completion-capable model reachable on this machine — environment fact, not a failure.');
|
||||||
|
}
|
||||||
|
|
||||||
|
userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-ai-index-test-'));
|
||||||
|
|
||||||
|
electronApp = await electron.launch({
|
||||||
|
args: [projectRoot, `--user-data-dir=${userDataDir}`],
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
VNCMAIL_TEST_FIXED_PORT: String(FIXED_PORT),
|
||||||
|
DEV_MOCK_JMAP: 'true',
|
||||||
|
JMAP_SERVER_URL: `${ORIGIN}/api/dev-jmap`,
|
||||||
|
SESSION_SECRET: 'electron-ai-local-index-verify-32-chars-min',
|
||||||
|
// Deliberately UNSET: isolates grounding to the local FTS leg (see
|
||||||
|
// module header) — the server embeddings leg 404s cleanly instead
|
||||||
|
// of silently also being able to answer the question.
|
||||||
|
AI_SERVER_BASE_URL: '',
|
||||||
|
NODE_ENV: 'production',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
appWindow = await electronApp.firstWindow();
|
||||||
|
await appWindow.waitForLoadState('domcontentloaded');
|
||||||
|
});
|
||||||
|
|
||||||
|
test.afterAll(async () => {
|
||||||
|
await electronApp?.close();
|
||||||
|
if (userDataDir) fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('logs in, builds the real encrypted index, and a local Ollama model answers a mail question grounded in it', async () => {
|
||||||
|
// ── 1. Real dev-mode login (sets the real session cookie the offline
|
||||||
|
// index and every other server-side-identity feature need). ──
|
||||||
|
const devLoginContainer = appWindow.locator('div', { hasText: 'Dev mode - logging in as dev@localhost' }).last();
|
||||||
|
await devLoginContainer.getByRole('button').click();
|
||||||
|
await appWindow.waitForURL((url) => !url.pathname.includes('login'), { timeout: 20000 });
|
||||||
|
|
||||||
|
// ── 2. Build the real encrypted local index: delta-sync the mock
|
||||||
|
// account's mail into the replica store, then write it into SQLite/FTS5.
|
||||||
|
// Chains /api/offline/sync while unfinishedWork is true, capped so a
|
||||||
|
// real bug can't hang the test forever. ──
|
||||||
|
const syncOutcome = await appWindow.evaluate(async () => {
|
||||||
|
let unfinished = true;
|
||||||
|
let calls = 0;
|
||||||
|
const statuses: number[] = [];
|
||||||
|
while (unfinished && calls < 10) {
|
||||||
|
const res = await fetch('/api/offline/sync', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' });
|
||||||
|
statuses.push(res.status);
|
||||||
|
if (!res.ok) break;
|
||||||
|
const body = await res.json();
|
||||||
|
unfinished = body.unfinishedWork === true;
|
||||||
|
calls++;
|
||||||
|
}
|
||||||
|
const reindexRes = await fetch('/api/offline/reindex', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ catchUp: true }) });
|
||||||
|
return { syncStatuses: statuses, syncCalls: calls, reindexStatus: reindexRes.status, reindexBody: await reindexRes.json().catch(() => null) };
|
||||||
|
});
|
||||||
|
console.log('[ai-local-index] sync+reindex outcome:', JSON.stringify(syncOutcome));
|
||||||
|
expect(syncOutcome.syncStatuses.every((s) => s === 200)).toBe(true);
|
||||||
|
expect(syncOutcome.reindexStatus).toBe(200);
|
||||||
|
|
||||||
|
// ── 3. Prove the local index itself is real and queryable BEFORE
|
||||||
|
// touching the LLM at all — isolates "is the SQLite/FTS5 index working"
|
||||||
|
// from "did the model use it correctly". ──
|
||||||
|
const directSearch = await appWindow.evaluate(async () => {
|
||||||
|
const res = await fetch(`/api/offline/search?q=${encodeURIComponent('Villa sul Lago check-in')}&limit=6`);
|
||||||
|
return { status: res.status, body: await res.json().catch(() => null) };
|
||||||
|
});
|
||||||
|
console.log('[ai-local-index] direct /api/offline/search result:', JSON.stringify(directSearch.body));
|
||||||
|
expect(directSearch.status, 'the encrypted local index must be reachable (200), not 404 (feature disabled) or 503 (no key channel)').toBe(200);
|
||||||
|
expect(directSearch.body?.ok).toBe(true);
|
||||||
|
const hitTitles = (directSearch.body?.hits ?? []).map((h: { title?: string }) => h.title ?? '');
|
||||||
|
expect(hitTitles.some((t: string) => /villa sul lago/i.test(t)), `expected a "Villa sul Lago" hit in the real index, got: ${JSON.stringify(hitTitles)}`).toBe(true);
|
||||||
|
|
||||||
|
// ── 4. Navigate to the real AI Assistant settings UI and use the
|
||||||
|
// local-discovery "Connect" banner — the exact flow a real user takes,
|
||||||
|
// proving discovery -> connect -> ask works as one integrated feature.
|
||||||
|
// Deliberately an in-app SPA navigation (click the real sidebar link),
|
||||||
|
// NOT appWindow.goto() — a full page reload drops whatever client-only
|
||||||
|
// session state the dev-mode login established (confirmed: goto('/settings')
|
||||||
|
// bounces straight back to /login even though the JMAP session cookie
|
||||||
|
// from step 2/3 is still valid), so the click is load-bearing, not
|
||||||
|
// cosmetic. ──
|
||||||
|
await appWindow.locator('a[href="/settings"], a[href*="/settings"]').first().click();
|
||||||
|
const searchBox = appWindow.locator('input[type="search"]').first();
|
||||||
|
await searchBox.fill('AI Assistant');
|
||||||
|
await appWindow.getByRole('button', { name: 'AI Assistant' }).click();
|
||||||
|
|
||||||
|
const connectButton = appWindow.getByRole('button', { name: /Connect/i });
|
||||||
|
await expect(connectButton, 'the local-discovery banner should appear since a real Ollama is running on this machine').toBeVisible({ timeout: 10000 });
|
||||||
|
const bannerText = await appWindow.locator('text=Local AI found on this machine').locator('..').innerText();
|
||||||
|
console.log('[ai-local-index] discovery banner text:', bannerText);
|
||||||
|
await connectButton.click();
|
||||||
|
|
||||||
|
// ── 5. Ask a question only answerable by combining the local LLM
|
||||||
|
// with the local index's actual content. ──
|
||||||
|
const questionBox = appWindow.getByPlaceholder(/What did legal say/i);
|
||||||
|
await questionBox.fill('When is check-in for the Villa sul Lago booking, and what time?');
|
||||||
|
const askButton = appWindow.getByRole('button', { name: /^Ask$/ });
|
||||||
|
await expect(askButton, 'Ask must be enabled immediately after Connect pre-fills provider+model').toBeEnabled({ timeout: 5000 });
|
||||||
|
|
||||||
|
const chatRequests: string[] = [];
|
||||||
|
const offlineSearchCalls: Array<{ url: string; status: number; body: unknown }> = [];
|
||||||
|
appWindow.on('request', (req) => {
|
||||||
|
if (req.url().includes('11434')) chatRequests.push(`${req.method()} ${req.url()}`);
|
||||||
|
});
|
||||||
|
appWindow.on('response', async (res) => {
|
||||||
|
if (res.url().includes('/api/offline/search')) {
|
||||||
|
offlineSearchCalls.push({ url: res.url(), status: res.status(), body: await res.json().catch(() => null) });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Log the exact prompt Ollama actually received, straight from the
|
||||||
|
// request body — the ground truth for "did retrieval even fire".
|
||||||
|
const ollamaChatPayloads: unknown[] = [];
|
||||||
|
await appWindow.route('**/api/chat', async (route) => {
|
||||||
|
try {
|
||||||
|
ollamaChatPayloads.push(JSON.parse(route.request().postData() ?? 'null'));
|
||||||
|
} catch { /* ignore parse failure, still let the request through */ }
|
||||||
|
await route.continue();
|
||||||
|
});
|
||||||
|
|
||||||
|
await askButton.click();
|
||||||
|
|
||||||
|
const answerLocator = appWindow.locator('p.whitespace-pre-wrap').first();
|
||||||
|
await expect(answerLocator, 'the local Ollama model should produce an answer within a generous timeout').toBeVisible({ timeout: 60000 });
|
||||||
|
const answerText = await answerLocator.innerText();
|
||||||
|
console.log('[ai-local-index] final answer:', answerText);
|
||||||
|
console.log('[ai-local-index] direct-to-Ollama requests observed:', chatRequests);
|
||||||
|
console.log('[ai-local-index] /api/offline/search calls during Ask:', JSON.stringify(offlineSearchCalls));
|
||||||
|
console.log('[ai-local-index] exact payload(s) sent to Ollama /api/chat:', JSON.stringify(ollamaChatPayloads));
|
||||||
|
|
||||||
|
// The real proof: the model's own words contain the fact that only
|
||||||
|
// exists in the indexed email (28 March, 15:00), and the request log
|
||||||
|
// shows the renderer talked to Ollama's loopback address directly.
|
||||||
|
expect(answerText).toMatch(/28\s*march|march\s*28/i);
|
||||||
|
expect(answerText).toMatch(/15:00|3\s*pm|3:00\s*pm/i);
|
||||||
|
expect(chatRequests.some((r) => r.includes('/api/chat')), `expected a direct renderer -> Ollama /api/chat request, saw: ${JSON.stringify(chatRequests)}`).toBe(true);
|
||||||
|
|
||||||
|
await appWindow.screenshot({ path: path.join(projectRoot, 'electron-ai-local-index-result.png'), fullPage: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
+12
-1
@@ -173,7 +173,18 @@ async function startStandaloneServer(): Promise<string> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const port = await getFreePort();
|
// Normally a random free port, chosen fresh every launch - JMAP_SERVER_URL
|
||||||
|
// never needs to reference it back (a real deployment's Stalwart lives at
|
||||||
|
// its own fixed address). VNCMAIL_TEST_FIXED_PORT is a narrow escape hatch
|
||||||
|
// for e2e tests that DO need to know the port ahead of time - specifically
|
||||||
|
// to point DEV_MOCK_JMAP's JMAP_SERVER_URL at this same standalone server's
|
||||||
|
// own /api/dev-jmap route, which is the only way to exercise the real
|
||||||
|
// encrypted offline index (lib/mail-index/**) without a real Stalwart
|
||||||
|
// fixture: that index's key channel only gets wired up in this function,
|
||||||
|
// never when ELECTRON_LOAD_URL bypasses it for a plain `next dev` target.
|
||||||
|
// Unset in every normal launch, so this changes nothing outside a test run.
|
||||||
|
const fixedPort = process.env.VNCMAIL_TEST_FIXED_PORT ? Number(process.env.VNCMAIL_TEST_FIXED_PORT) : null;
|
||||||
|
const port = fixedPort && Number.isInteger(fixedPort) ? fixedPort : await getFreePort();
|
||||||
const url = `http://127.0.0.1:${port}`;
|
const url = `http://127.0.0.1:${port}`;
|
||||||
|
|
||||||
const storeDir = getIndexStoreDir();
|
const storeDir = getIndexStoreDir();
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
|
||||||
|
import {
|
||||||
|
discoverLocalOllama,
|
||||||
|
recommendDefaultModel,
|
||||||
|
largestModel,
|
||||||
|
isLocalDiscoveryDismissed,
|
||||||
|
dismissLocalDiscovery,
|
||||||
|
type DiscoveredLocalModel,
|
||||||
|
} from '../local-discovery';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Real model list from this machine's Ollama (`curl 127.0.0.1:11434/api/tags`,
|
||||||
|
* 2026-08-06) — used as the test fixture rather than invented data, per the
|
||||||
|
* explicit instruction to use the real local runtime as the test case for
|
||||||
|
* "which queries are required and how to add most of the modules
|
||||||
|
* automatically". Sizes/params/capabilities are copied verbatim.
|
||||||
|
*/
|
||||||
|
const REAL_MACHINE_MODELS: DiscoveredLocalModel[] = [
|
||||||
|
{ name: 'nomic-embed-text:latest', capabilities: ['embedding'], parameterSize: '137M', sizeBytes: 274_302_450 },
|
||||||
|
{ name: 'qwen2.5:32b', capabilities: ['completion', 'tools'], parameterSize: '32.8B', sizeBytes: 19_851_349_669 },
|
||||||
|
{ name: 'gemma4:12b-mlx', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '', sizeBytes: 9_977_519_169 },
|
||||||
|
{ name: 'gemma4:latest', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '8.0B', sizeBytes: 9_608_350_718 },
|
||||||
|
{ name: 'deepseek-r1:32b', capabilities: ['completion', 'thinking'], parameterSize: '32.8B', sizeBytes: 19_851_337_809 },
|
||||||
|
{ name: 'deepseek-r1:14b', capabilities: ['completion', 'thinking'], parameterSize: '14.8B', sizeBytes: 8_988_112_209 },
|
||||||
|
{ name: 'llama3.2:latest', capabilities: ['completion', 'tools'], parameterSize: '3.2B', sizeBytes: 2_019_393_189 },
|
||||||
|
{ name: 'hermes3:8b', capabilities: ['completion', 'tools'], parameterSize: '8B', sizeBytes: 4_661_227_000 },
|
||||||
|
{ name: 'qwen3:latest', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '8.2B', sizeBytes: 5_200_000_000 },
|
||||||
|
{ name: 'gemma4:e4b', capabilities: ['completion', 'tools', 'thinking'], parameterSize: '8.0B', sizeBytes: 9_600_000_000 },
|
||||||
|
{ name: 'qwen3.5:latest', capabilities: ['vision', 'completion', 'tools', 'thinking'], parameterSize: '9.7B', sizeBytes: 6_600_000_000 },
|
||||||
|
];
|
||||||
|
|
||||||
|
describe('recommendDefaultModel', () => {
|
||||||
|
it('picks the smallest non-"thinking" chat model from a real mixed fleet', () => {
|
||||||
|
// llama3.2 (2.0GB) is the smallest completion-capable, non-reasoning
|
||||||
|
// model on this real machine — everything smaller is embedding-only.
|
||||||
|
expect(recommendDefaultModel(REAL_MACHINE_MODELS)).toBe('llama3.2:latest');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('never recommends an embedding-only model', () => {
|
||||||
|
const onlyEmbedding = [REAL_MACHINE_MODELS[0]]; // nomic-embed-text
|
||||||
|
expect(recommendDefaultModel(onlyEmbedding)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to the smallest "thinking" model when nothing else qualifies', () => {
|
||||||
|
const onlyReasoning = REAL_MACHINE_MODELS.filter((m) => m.capabilities.includes('thinking') && !m.capabilities.includes('vision'));
|
||||||
|
// Smallest of the thinking-only pool here is qwen3 (5.2GB) before gemma4 variants.
|
||||||
|
expect(recommendDefaultModel(onlyReasoning)).toBe('qwen3:latest');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null when no models are chat-capable at all', () => {
|
||||||
|
expect(recommendDefaultModel([])).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('largestModel', () => {
|
||||||
|
it('picks the biggest chat-capable model — qwen2.5:32b, by 11,860 bytes over deepseek-r1:32b', () => {
|
||||||
|
// Both are ~19.85GB on this real machine (same base size class), but
|
||||||
|
// qwen2.5:32b's actual manifest is very slightly larger — not a tie.
|
||||||
|
expect(largestModel(REAL_MACHINE_MODELS)).toBe('qwen2.5:32b');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('excludes embedding-only models even though they can be tiny or huge', () => {
|
||||||
|
expect(largestModel([REAL_MACHINE_MODELS[0]])).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('discoverLocalOllama', () => {
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses a real-shaped /api/tags response into DiscoveredLocalModel[]', async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({
|
||||||
|
models: [
|
||||||
|
{ name: 'llama3.2:latest', capabilities: ['completion', 'tools'], size: 2_019_393_189, details: { parameter_size: '3.2B' } },
|
||||||
|
{ name: 'nomic-embed-text:latest', capabilities: ['embedding'], size: 274_302_450, details: { parameter_size: '137M' } },
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
const result = await discoverLocalOllama(['http://127.0.0.1:11434']);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.baseUrl).toBe('http://127.0.0.1:11434');
|
||||||
|
expect(result?.models).toHaveLength(2);
|
||||||
|
expect(result?.models[0]).toEqual({
|
||||||
|
name: 'llama3.2:latest',
|
||||||
|
capabilities: ['completion', 'tools'],
|
||||||
|
parameterSize: '3.2B',
|
||||||
|
sizeBytes: 2_019_393_189,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requires exactly one query — a single /api/tags call, no follow-up /api/show requests', async () => {
|
||||||
|
const fetchMock = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ models: [{ name: 'llama3.2:latest', capabilities: ['completion'], size: 1, details: {} }] }),
|
||||||
|
});
|
||||||
|
global.fetch = fetchMock as unknown as typeof fetch;
|
||||||
|
|
||||||
|
await discoverLocalOllama(['http://127.0.0.1:11434']);
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(fetchMock).toHaveBeenCalledWith('http://127.0.0.1:11434/api/tags', expect.anything());
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls through to the next candidate base URL when the first is unreachable', async () => {
|
||||||
|
const fetchMock = vi.fn()
|
||||||
|
.mockRejectedValueOnce(new Error('connection refused'))
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ models: [{ name: 'llama3.2:latest', capabilities: ['completion'], size: 1, details: {} }] }),
|
||||||
|
});
|
||||||
|
global.fetch = fetchMock as unknown as typeof fetch;
|
||||||
|
|
||||||
|
const result = await discoverLocalOllama(['http://127.0.0.1:11434', 'http://localhost:11434']);
|
||||||
|
expect(result?.baseUrl).toBe('http://localhost:11434');
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null when nothing answers on any candidate', async () => {
|
||||||
|
global.fetch = vi.fn().mockRejectedValue(new Error('connection refused')) as unknown as typeof fetch;
|
||||||
|
const result = await discoverLocalOllama(['http://127.0.0.1:11434', 'http://localhost:11434']);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null (not an empty-models result) when Ollama answers with zero models installed', async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ models: [] }) }) as unknown as typeof fetch;
|
||||||
|
const result = await discoverLocalOllama(['http://127.0.0.1:11434']);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('dismissal persistence', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
window.localStorage.clear();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is not dismissed by default', () => {
|
||||||
|
expect(isLocalDiscoveryDismissed()).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('persists a dismissal across calls', () => {
|
||||||
|
dismissLocalDiscovery();
|
||||||
|
expect(isLocalDiscoveryDismissed()).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
// Local-LLM auto-discovery: probes the loopback addresses a local Ollama
|
||||||
|
// normally binds to, and — if one answers — recommends a model to connect
|
||||||
|
// with, so a user with Ollama already running never has to type a base URL
|
||||||
|
// or a model name by hand.
|
||||||
|
//
|
||||||
|
// One network call is enough: Ollama's own `/api/tags` already reports
|
||||||
|
// per-model `capabilities` (completion/embedding/tools/thinking/vision),
|
||||||
|
// `size`, and `details.parameter_size` — everything the recommendation
|
||||||
|
// heuristic below needs, with no follow-up `/api/show` round trips.
|
||||||
|
|
||||||
|
export interface DiscoveredLocalModel {
|
||||||
|
name: string;
|
||||||
|
capabilities: string[];
|
||||||
|
parameterSize: string;
|
||||||
|
sizeBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LocalDiscoveryResult {
|
||||||
|
baseUrl: string;
|
||||||
|
models: DiscoveredLocalModel[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OllamaTagsResponse {
|
||||||
|
models?: Array<{
|
||||||
|
name: string;
|
||||||
|
capabilities?: string[];
|
||||||
|
size?: number;
|
||||||
|
details?: { parameter_size?: string };
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ollama's own default bind address, plus the hostname form — some setups
|
||||||
|
// (notably OLLAMA_ORIGINS-restricted CORS allowlists keyed by hostname
|
||||||
|
// rather than IP) answer one but not the other.
|
||||||
|
const DEFAULT_PROBE_URLS = ['http://127.0.0.1:11434', 'http://localhost:11434'];
|
||||||
|
const PROBE_TIMEOUT_MS = 1200;
|
||||||
|
|
||||||
|
async function probeOne(baseUrl: string): Promise<LocalDiscoveryResult | null> {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${baseUrl}/api/tags`, { signal: controller.signal });
|
||||||
|
if (!res.ok) return null;
|
||||||
|
const body = (await res.json()) as OllamaTagsResponse;
|
||||||
|
const models = (body.models ?? [])
|
||||||
|
.filter((m) => typeof m.name === 'string' && m.name)
|
||||||
|
.map((m) => ({
|
||||||
|
name: m.name,
|
||||||
|
capabilities: m.capabilities ?? [],
|
||||||
|
parameterSize: m.details?.parameter_size ?? '',
|
||||||
|
sizeBytes: m.size ?? 0,
|
||||||
|
}));
|
||||||
|
return models.length > 0 ? { baseUrl, models } : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Tries each candidate in turn (not in parallel — the common case is the
|
||||||
|
* first one answering, and probing sequentially avoids a burst of
|
||||||
|
* simultaneous loopback connection attempts for no benefit). */
|
||||||
|
export async function discoverLocalOllama(
|
||||||
|
candidateBaseUrls: readonly string[] = DEFAULT_PROBE_URLS,
|
||||||
|
): Promise<LocalDiscoveryResult | null> {
|
||||||
|
for (const baseUrl of candidateBaseUrls) {
|
||||||
|
const result = await probeOne(baseUrl);
|
||||||
|
if (result) return result;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Picks one sensible default out of whatever's installed, so "Connect"
|
||||||
|
* needs no follow-up decision. Chat-capable models only (never an
|
||||||
|
* embedding-only model like nomic-embed-text). Among those, prefers
|
||||||
|
* non-"thinking" models — a reasoning model's chain-of-thought preamble
|
||||||
|
* reads as a broken first response in a guided setup, however good the
|
||||||
|
* final answer is — and then the smallest by download size, on the theory
|
||||||
|
* that the fastest first reply makes the best first impression; a user who
|
||||||
|
* wants the largest/most capable model for real work can still pick it from
|
||||||
|
* the full list this only pre-selects.
|
||||||
|
*/
|
||||||
|
export function recommendDefaultModel(models: readonly DiscoveredLocalModel[]): string | null {
|
||||||
|
const chatCapable = models.filter((m) => m.capabilities.includes('completion'));
|
||||||
|
if (chatCapable.length === 0) return null;
|
||||||
|
const nonReasoning = chatCapable.filter((m) => !m.capabilities.includes('thinking'));
|
||||||
|
const pool = nonReasoning.length > 0 ? nonReasoning : chatCapable;
|
||||||
|
return [...pool].sort((a, b) => a.sizeBytes - b.sizeBytes)[0].name;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The largest chat-capable model, for the "most capable" callout next to
|
||||||
|
* the speed-optimized recommendation above — skipped in the UI when it's
|
||||||
|
* the same model `recommendDefaultModel` already picked. */
|
||||||
|
export function largestModel(models: readonly DiscoveredLocalModel[]): string | null {
|
||||||
|
const chatCapable = models.filter((m) => m.capabilities.includes('completion'));
|
||||||
|
if (chatCapable.length === 0) return null;
|
||||||
|
return [...chatCapable].sort((a, b) => b.sizeBytes - a.sizeBytes)[0].name;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DISMISSED_KEY = 'vncmail:ai:local-discovery-dismissed';
|
||||||
|
|
||||||
|
export function isLocalDiscoveryDismissed(): boolean {
|
||||||
|
if (typeof window === 'undefined') return true;
|
||||||
|
return window.localStorage.getItem(DISMISSED_KEY) === 'true';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function dismissLocalDiscovery(): void {
|
||||||
|
if (typeof window === 'undefined') return;
|
||||||
|
window.localStorage.setItem(DISMISSED_KEY, 'true');
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|||||||
import { isSqlcipherAvailable } from '../binding';
|
import { isSqlcipherAvailable } from '../binding';
|
||||||
import type { SqlcipherConstructor, SqlcipherDatabase, SqlcipherStatement } from '../binding';
|
import type { SqlcipherConstructor, SqlcipherDatabase, SqlcipherStatement } from '../binding';
|
||||||
import { accountFileToken, getStoreDir, indexDbPath, STORE_DIR_ENV } from '../paths';
|
import { accountFileToken, getStoreDir, indexDbPath, STORE_DIR_ENV } from '../paths';
|
||||||
import { MailIndex, openKeyed, toFtsMatchQuery, type IndexDoc } from '../store';
|
import { MailIndex, openKeyed, toFtsMatchQuery, toFtsMatchQueryAny, type IndexDoc } from '../store';
|
||||||
|
|
||||||
describe('toFtsMatchQuery', () => {
|
describe('toFtsMatchQuery', () => {
|
||||||
it('quotes every token so FTS5 operators in user input cannot break the query', () => {
|
it('quotes every token so FTS5 operators in user input cannot break the query', () => {
|
||||||
@@ -48,6 +48,62 @@ describe('toFtsMatchQuery', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('toFtsMatchQueryAny', () => {
|
||||||
|
it('drops English function words and OR-joins what is left - the confirmed-live failure this fixes', () => {
|
||||||
|
// AND-every-token (toFtsMatchQuery) returns 0 hits for this exact
|
||||||
|
// question against a document that only contains "Villa sul Lago" and
|
||||||
|
// "check-in" - see app/api/offline/search/route.ts's comment and the
|
||||||
|
// e2e electron-ai-local-index.spec.ts run that first caught this.
|
||||||
|
const result = toFtsMatchQueryAny('When is check-in for the Villa sul Lago booking, and what time?');
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result).not.toContain(' AND ');
|
||||||
|
expect(result).toContain('"check-in"');
|
||||||
|
expect(result).toContain('"Villa"');
|
||||||
|
expect(result).toContain('"sul"');
|
||||||
|
expect(result).toContain('"Lago"');
|
||||||
|
expect(result).toContain('"booking"');
|
||||||
|
// "time" is the last surviving content word, so it gets the
|
||||||
|
// prefix-match star - not "Lago", which is merely the last one this
|
||||||
|
// test happens to name first.
|
||||||
|
expect(result).toContain('"time"*');
|
||||||
|
// Pure stop words, correctly dropped rather than OR-joined as noise that
|
||||||
|
// would otherwise match almost every document in a mailbox.
|
||||||
|
expect(result).not.toMatch(/"When"|"is"|"for"|"the"|"and"|"what"/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to the unfiltered text when every word is a stop word, rather than searching for nothing', () => {
|
||||||
|
// "What is this" is 100% stop words - dropping all of them would leave
|
||||||
|
// zero tokens (a null match, meaning "return everything" is wrong for a
|
||||||
|
// question shaped like this); falling back to the original text at
|
||||||
|
// least keeps a real, if weak, query.
|
||||||
|
const result = toFtsMatchQueryAny('What is this');
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('still safely quotes FTS5 syntax characters even after stop-word filtering removes the surrounding noise', () => {
|
||||||
|
// "OR"/"NEAR" themselves are common enough as English words that this
|
||||||
|
// builder's stop-word list intentionally drops bare "or" (unlike
|
||||||
|
// toFtsMatchQuery, which preserves it verbatim - see that test's own
|
||||||
|
// comment on why: different concern, different guarantee). The safety
|
||||||
|
// property that DOES still apply here is the one that matters for a
|
||||||
|
// 500: whatever tokens survive filtering are always quoted before
|
||||||
|
// reaching FTS5, so a stray `"`/`*`/`(` in real question text can never
|
||||||
|
// raise a syntax error.
|
||||||
|
const result = toFtsMatchQueryAny('a" NEAR(bar) baz*');
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result).toContain('"NEAR"');
|
||||||
|
expect(result).toContain('"bar"');
|
||||||
|
expect(result).toContain('"baz"');
|
||||||
|
expect(result).not.toMatch(/fts5|syntax/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null for input with no usable tokens', () => {
|
||||||
|
expect(toFtsMatchQueryAny('')).toBeNull();
|
||||||
|
expect(toFtsMatchQueryAny('***')).toBeNull();
|
||||||
|
expect(toFtsMatchQueryAny(undefined as unknown as string)).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('paths', () => {
|
describe('paths', () => {
|
||||||
const original = process.env[STORE_DIR_ENV];
|
const original = process.env[STORE_DIR_ENV];
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
|||||||
+79
-24
@@ -331,8 +331,12 @@ export class MailIndex {
|
|||||||
types?: readonly ContentType[];
|
types?: readonly ContentType[];
|
||||||
limit?: number;
|
limit?: number;
|
||||||
snippetTokens?: number;
|
snippetTokens?: number;
|
||||||
|
/** 'and' (default): every token required - a deliberate search-box query.
|
||||||
|
* 'any': stop words dropped, remaining tokens OR-joined, ranked by bm25 -
|
||||||
|
* a natural-language question (see toFtsMatchQueryAny's docstring). */
|
||||||
|
mode?: 'and' | 'any';
|
||||||
}): SearchHit[] {
|
}): SearchHit[] {
|
||||||
const match = toFtsMatchQuery(opts.query);
|
const match = opts.mode === 'any' ? toFtsMatchQueryAny(opts.query) : toFtsMatchQuery(opts.query);
|
||||||
if (!match) return [];
|
if (!match) return [];
|
||||||
|
|
||||||
const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200);
|
const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200);
|
||||||
@@ -446,34 +450,85 @@ function safeParseObject(v: unknown): Record<string, unknown> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Turns arbitrary user text into a safe FTS5 MATCH expression.
|
* Splits and safely quotes raw text into FTS5-safe tokens, shared by both
|
||||||
*
|
* query-builders below. Split on anything that isn't a word character or an
|
||||||
* FTS5's query syntax is not SQL, so parameter binding does NOT protect it: a
|
* intra-word mark - keeps unicode letters (so "Müller" and "東京" survive)
|
||||||
* bare `"` or a stray `*`/`NEAR`/`:` in user input raises
|
* via the u flag. Every token is quoted (making it a literal phrase) so a
|
||||||
* `fts5: syntax error`, which would turn a normal search box into a 500. Every
|
* bare `"` or a stray `*`/`NEAR`/`:` in user input can never raise FTS5's own
|
||||||
* token is quoted (making it a literal phrase) and a trailing `*` is added to
|
* `fts5: syntax error` - that would turn a normal search into a 500.
|
||||||
* the last token so typing continues to match as the user types.
|
*/
|
||||||
|
function quoteFtsTokens(raw: string): string[] {
|
||||||
|
return raw
|
||||||
|
.normalize('NFC')
|
||||||
|
.split(/[^\p{L}\p{N}_@.'-]+/u)
|
||||||
|
.map((t) => t.replace(/^['-]+|['-]+$/g, ''))
|
||||||
|
.filter((t) => t.length > 0)
|
||||||
|
.slice(0, 24)
|
||||||
|
.map((t, i, all) => {
|
||||||
|
const quoted = `"${t.replace(/"/g, '""')}"`;
|
||||||
|
// Prefix-match only the final token, and only if it's long enough to not
|
||||||
|
// match half the mailbox.
|
||||||
|
return i === all.length - 1 && t.length >= 3 ? `${quoted}*` : quoted;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turns arbitrary user text into a safe FTS5 MATCH expression, every token
|
||||||
|
* required (AND-joined). Right for a deliberate, short search-box query,
|
||||||
|
* where requiring every word is what makes results precise as you type.
|
||||||
*
|
*
|
||||||
* Exported for unit testing - it is the one piece of this file with no
|
* Exported for unit testing - it is the one piece of this file with no
|
||||||
* database dependency and the most ways to be wrong.
|
* database dependency and the most ways to be wrong.
|
||||||
*/
|
*/
|
||||||
export function toFtsMatchQuery(raw: string): string | null {
|
export function toFtsMatchQuery(raw: string): string | null {
|
||||||
if (typeof raw !== 'string') return null;
|
if (typeof raw !== 'string') return null;
|
||||||
// Split on anything that isn't a word character or an intra-word mark. Keeps
|
const tokens = quoteFtsTokens(raw);
|
||||||
// unicode letters (so "Müller" and "東京" survive) via the u flag.
|
|
||||||
const tokens = raw
|
|
||||||
.normalize('NFC')
|
|
||||||
.split(/[^\p{L}\p{N}_@.'-]+/u)
|
|
||||||
.map((t) => t.replace(/^['-]+|['-]+$/g, ''))
|
|
||||||
.filter((t) => t.length > 0)
|
|
||||||
.slice(0, 24);
|
|
||||||
if (tokens.length === 0) return null;
|
if (tokens.length === 0) return null;
|
||||||
return tokens
|
return tokens.join(' AND ');
|
||||||
.map((t, i) => {
|
}
|
||||||
const quoted = `"${t.replace(/"/g, '""')}"`;
|
|
||||||
// Prefix-match only the final token, and only if it's long enough to not
|
// A minimal, well-known set of English function words that carry no
|
||||||
// match half the mailbox.
|
// retrieval signal - kept out of toFtsMatchQueryAny's OR expression so they
|
||||||
return i === tokens.length - 1 && t.length >= 3 ? `${quoted}*` : quoted;
|
// don't drown out the bm25 ranking's actual signal (see below). Deliberately
|
||||||
})
|
// NOT applied inside quoteFtsTokens/toFtsMatchQuery: that function's own
|
||||||
.join(' AND ');
|
// tests rely on "AND"/"OR"/"NOT" surviving verbatim as literal search terms
|
||||||
|
// (FTS5-keyword-injection safety) - a different concern from this one's job
|
||||||
|
// of turning a natural-language QUESTION into a good search.
|
||||||
|
const RETRIEVAL_STOP_WORDS = new Set([
|
||||||
|
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'am',
|
||||||
|
'and', 'or', 'but', 'if', 'then', 'than', 'so', 'because',
|
||||||
|
'for', 'of', 'to', 'in', 'on', 'at', 'by', 'with', 'from', 'as', 'about', 'into', 'over', 'after', 'before',
|
||||||
|
'that', 'this', 'these', 'those', 'what', 'when', 'where', 'who', 'whom', 'which', 'why', 'how',
|
||||||
|
'do', 'does', 'did', 'doing', 'done',
|
||||||
|
'can', 'could', 'will', 'would', 'shall', 'should', 'may', 'might', 'must',
|
||||||
|
'i', 'you', 'he', 'she', 'it', 'we', 'they', 'my', 'your', 'his', 'her', 'its', 'our', 'their', 'me', 'him', 'us', 'them',
|
||||||
|
'not', 'no',
|
||||||
|
]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turns a natural-language QUESTION into a lenient FTS5 MATCH expression:
|
||||||
|
* stop words dropped, remaining tokens OR-joined so bm25 ranks by how many
|
||||||
|
* content words matched instead of requiring every one of them present.
|
||||||
|
*
|
||||||
|
* toFtsMatchQuery's strict AND is wrong for this shape of input: a real
|
||||||
|
* question like "When is check-in for the Villa sul Lago booking?" shares
|
||||||
|
* almost none of its own function words ("when"/"is"/"for"/"the") with the
|
||||||
|
* document that actually answers it, so ANDing every token together returns
|
||||||
|
* nothing - confirmed live: 0 hits for the full question, 2 correct hits for
|
||||||
|
* the same index once reduced to "Villa sul Lago check-in". The one real
|
||||||
|
* caller of `/api/offline/search?q=...` is exactly this AI-question shape
|
||||||
|
* (see that route's own header - no manual search-box UI hits it today), so
|
||||||
|
* this is the query builder that route now uses, not toFtsMatchQuery.
|
||||||
|
*/
|
||||||
|
export function toFtsMatchQueryAny(raw: string): string | null {
|
||||||
|
if (typeof raw !== 'string') return null;
|
||||||
|
const withoutStopWords = raw
|
||||||
|
.split(/\s+/)
|
||||||
|
.filter((w) => w.length > 0 && !RETRIEVAL_STOP_WORDS.has(w.toLowerCase().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '')))
|
||||||
|
.join(' ');
|
||||||
|
// Every word was a stop word (e.g. "What is this?") - fall back to the
|
||||||
|
// original text rather than searching for literally nothing.
|
||||||
|
const tokens = quoteFtsTokens(withoutStopWords.length > 0 ? withoutStopWords : raw);
|
||||||
|
if (tokens.length === 0) return null;
|
||||||
|
return tokens.join(' OR ');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { defineConfig } from '@playwright/test';
|
||||||
|
|
||||||
|
// Separate from playwright.electron.config.ts (which hardcodes testMatch to
|
||||||
|
// electron-smoke.spec.ts) purely so this one test can get a longer timeout —
|
||||||
|
// real Ollama inference plus a real multi-round offline sync/reindex chain
|
||||||
|
// legitimately takes longer than the smoke suite's 60s budget.
|
||||||
|
export default defineConfig({
|
||||||
|
testDir: './e2e',
|
||||||
|
testMatch: 'electron-ai-local-index.spec.ts',
|
||||||
|
timeout: 120000,
|
||||||
|
retries: 0,
|
||||||
|
use: {
|
||||||
|
trace: 'retain-on-failure',
|
||||||
|
},
|
||||||
|
workers: 1,
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user