From 57c06e38a42a7ffadea180b821797710550d5daa Mon Sep 17 00:00:00 2001 From: vncmail-ci Date: Thu, 6 Aug 2026 18:22:13 +0000 Subject: [PATCH 1/6] chore(deploy): pin dev to sha-f6fc34fa [skip ci] --- deploy/k8s/overlays/dev/image-tag/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml index f7fb6efe..8770c469 100644 --- a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml +++ b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml @@ -5,4 +5,4 @@ kind: Component images: - name: vncmail-plus newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus - newTag: sha-1f199fdc + newTag: sha-f6fc34fa From 87336981d3885bff37eb7cb099aa250a05f63d44 Mon Sep 17 00:00:00 2001 From: Bernd Rodler Date: Fri, 7 Aug 2026 09:45:10 +0200 Subject: [PATCH 2/6] feat(mail-index): 1-year retention by default + a recency retrieval leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two things that made real questions fail against a correctly-populated index, both fixed at the root. RETENTION (A1). `INDEX_WINDOW_DAYS = 30` was not merely a fetch bound — catch-up also PRUNED mail older than it, so "summarise everything from July" was unanswerable in August because the rows had been deleted, while the UI said only that nothing matched. Now a user-visible setting (Settings → About & Data): 30 days / 3 months / 1 year / everything, defaulting to 1 YEAR per the product owner. The window bounds the fetch AND the prune from one value so the two can never disagree and delete what was just written; "everything" skips pruning entirely rather than falling back to some default bound. The per-pass ceiling scales with the window (500/30d, hard cap 20k) because 500 messages is right for a month and nonsense for "everything". Email/query now omits the `after` filter entirely when unbounded — Stalwart rejects a malformed filter rather than treating `undefined` as unset. RECENCY (A2). Keyword search structurally cannot answer a question about WHEN: bm25 ranks by term overlap, so "who sent the last email" matches documents containing the word "last", and "all mails in July" matches documents containing "July" — not documents dated in July. Both were asked by a real user and both failed. New lib/mail-index/recency.ts detects time intent (English + German, since the UI ships German) and turns it into a date RANGE; new MailIndex.recent() answers it with an ordered scan over the already-indexed `occurred_at`. The route ADDS these hits to the keyword hits rather than replacing them — "what did the last mail from Anna say" is both kinds of question at once. Timezone subtlety worth knowing: bounds are built from LOCAL calendar boundaries and serialised as UTC instants, so "July" covers the user's July. A mail at 00:30 local on 1 July belongs to it even though its stored UTC timestamp reads 30 June. My first test asserted the ISO string prefix, which would have enshrined the opposite and passed only in UTC — the tests now assert the local-time property instead. SCOPE, stated by the product owner and now enforced structurally: the assistant only ever sees the mailbox the user is signed in to. Both retrieval legs resolve the active account (local leg by cookie slot, server leg by the session's own JMAP account); there is deliberately no fan-out across connected or shared mailboxes, and adding one would be a policy change, not a feature. Gate: tsc clean, eslint clean, 2520/2520 tests (8 new for recency intent), build clean. --- app/(main)/[locale]/page.tsx | 3 +- app/api/offline/reindex/route.ts | 6 +- app/api/offline/search/route.ts | 29 +++++- components/settings/local-index-settings.tsx | 36 +++++++- lib/ai/local-client.ts | 9 ++ lib/mail-index-client.ts | 32 ++++++- lib/mail-index/__tests__/recency.test.ts | 75 ++++++++++++++++ lib/mail-index/jmap.ts | 8 +- lib/mail-index/recency.ts | 92 ++++++++++++++++++++ lib/mail-index/reindex.ts | 75 +++++++++++++--- lib/mail-index/store.ts | 63 ++++++++++++++ 11 files changed, 406 insertions(+), 22 deletions(-) create mode 100644 lib/mail-index/__tests__/recency.test.ts create mode 100644 lib/mail-index/recency.ts diff --git a/app/(main)/[locale]/page.tsx b/app/(main)/[locale]/page.tsx index c7aab870..714868fa 100644 --- a/app/(main)/[locale]/page.tsx +++ b/app/(main)/[locale]/page.tsx @@ -1088,9 +1088,10 @@ export default function Home() { const runCatchUp = async (attempt: number) => { if (catchUpCancelled) return; try { - const { catchUpIndex } = await import('@/lib/mail-index-client'); + const { catchUpIndex, getRetentionDays } = await import('@/lib/mail-index-client'); const result = await catchUpIndex( useAccountStore.getState().getActiveAccount()?.cookieSlot, + getRetentionDays(), ); if (!result.ok && !result.unavailable && attempt + 1 < catchUpRetryDelaysMs.length) { catchUpTimer = setTimeout(() => void runCatchUp(attempt + 1), catchUpRetryDelaysMs[attempt + 1]); diff --git a/app/api/offline/reindex/route.ts b/app/api/offline/reindex/route.ts index 93ca0843..ef500b0c 100644 --- a/app/api/offline/reindex/route.ts +++ b/app/api/offline/reindex/route.ts @@ -17,7 +17,7 @@ import { isSqlcipherAvailable } from '@/lib/mail-index/binding'; import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key'; import { getStoreDir } from '@/lib/mail-index/paths'; import { - IndexSessionError, MAX_IDS_PER_CALL, resolveIndexSession, runIndex, + IndexSessionError, MAX_IDS_PER_CALL, normalizeWindowDays, resolveIndexSession, runIndex, type IndexRequest, } from '@/lib/mail-index/reindex'; import { CONTENT_TYPES, isContentType, type ContentType } from '@/lib/mail-index/store'; @@ -73,6 +73,10 @@ export async function POST(request: NextRequest) { removed: parseIdMap(body.removed), // Pruning is a catch-up concern; a single-delivery call shouldn't scan. prune: body.catchUp === true, + // `undefined` (absent) means "use the default"; an explicit null means + // keep everything. normalizeWindowDays() in runIndex clamps anything + // unexpected, since this value drives deletion. + windowDays: body.windowDays === undefined ? undefined : normalizeWindowDays(body.windowDays), }; try { diff --git a/app/api/offline/search/route.ts b/app/api/offline/search/route.ts index 3de06365..d6e33e91 100644 --- a/app/api/offline/search/route.ts +++ b/app/api/offline/search/route.ts @@ -17,6 +17,7 @@ import { IndexSessionError, resolveIndexSession } from '@/lib/mail-index/reindex import { isContentType, MailIndex, MailIndexUnavailableError, type ContentType, type SearchHit, } from '@/lib/mail-index/store'; +import { detectRecencyIntent } from '@/lib/mail-index/recency'; export const runtime = 'nodejs'; export const dynamic = 'force-dynamic'; @@ -84,7 +85,29 @@ export async function GET(request: NextRequest) { // not deliberate search-box keywords, so strict AND-every-token // matching (the default) drops nearly all of them. See // toFtsMatchQueryAny's docstring for the confirmed-live failure. - return { hits: index.search({ query, types, limit, mode: 'any' }), stats: wantStats ? stats : undefined }; + const keywordHits = index.search({ query, types, limit, mode: 'any' }); + + // RECENCY leg. Keyword search structurally cannot answer "the last + // mail" or "everything from July" (see lib/mail-index/recency.ts), so + // when the question is really about time, add a date-ordered slice. + // ADDED to the keyword hits rather than replacing them: "what did the + // last mail from Anna say" is both a time question and a content one. + const intent = detectRecencyIntent(query); + if (!intent) { + return { hits: keywordHits, stats: wantStats ? stats : undefined }; + } + const recentHits = index.recent({ + types, limit: Math.min(intent.limit, limit * 3), since: intent.since, until: intent.until, + }); + const seen = new Set(keywordHits.map((h) => `${h.contentType}:${h.id}`)); + const merged = [...keywordHits]; + for (const hit of recentHits) { + const key = `${hit.contentType}:${hit.id}`; + if (seen.has(key)) continue; + seen.add(key); + merged.push(hit); + } + return { hits: merged, stats: wantStats ? stats : undefined, recency: intent }; } finally { index.close(); } @@ -100,6 +123,10 @@ export async function GET(request: NextRequest) { // Everything a prompt needs, pre-joined in rank order. contextBlock: payload.hits.map(toContextBlock).join('\n\n---\n\n'), ...(payload.stats ? { stats: payload.stats } : {}), + // Present when the question was read as a time question — lets the + // client say "these are the newest N" instead of implying relevance + // ranking it did not do. + ...(payload.recency ? { recency: payload.recency } : {}), }, { headers: { 'Cache-Control': 'no-store' } }, ); diff --git a/components/settings/local-index-settings.tsx b/components/settings/local-index-settings.tsx index 59e529ce..76c88d36 100644 --- a/components/settings/local-index-settings.tsx +++ b/components/settings/local-index-settings.tsx @@ -14,7 +14,9 @@ import { Button } from '@/components/ui/button'; import { SettingsSection, SettingItem } from './settings-section'; import { isElectronShell } from '@/lib/electron-bridge'; import { useAccountStore } from '@/stores/account-store'; -import { catchUpIndex, fetchIndexStats, type IndexStats } from '@/lib/mail-index-client'; +import { + catchUpIndex, fetchIndexStats, getRetentionDays, setRetentionDays, type IndexStats, +} from '@/lib/mail-index-client'; import { chainSync, fetchReplicaStatus, purgeReplica, updateRetentionPolicy, type ReplicaStatus, type RetentionPolicy, @@ -35,6 +37,10 @@ export function LocalIndexSettings() { // `null` until the first probe resolves, so we don't flash a panel that then // vanishes on a non-desktop build. const [available, setAvailable] = useState(null); + // `null` = keep everything. Read once on mount; the setter writes through. + const [retentionDays, setRetentionDaysState] = useState(365); + + useEffect(() => { setRetentionDaysState(getRetentionDays()); }, []); const refreshStats = useCallback(async () => { const next = await fetchIndexStats(slot); @@ -54,7 +60,7 @@ export function LocalIndexSettings() { setBusy(true); setMessage(null); try { - const result = await catchUpIndex(slot); + const result = await catchUpIndex(slot, retentionDays); if (result.unavailable) { setAvailable(false); setMessage(result.error ?? 'The encrypted index is unavailable on this system.'); @@ -110,6 +116,32 @@ export function LocalIndexSettings() { {total} + + + + { const [local, server] = await Promise.all([fetchLocalLeg(question, slot), fetchServerLeg(question)]); lastLocalIndexReachable = local.indexReachable; diff --git a/lib/mail-index-client.ts b/lib/mail-index-client.ts index 2a1e09d2..9c0fd08e 100644 --- a/lib/mail-index-client.ts +++ b/lib/mail-index-client.ts @@ -70,6 +70,9 @@ export interface IndexRequestOptions { ids?: Partial>; /** Backfill the recent window for every supported type, and prune. */ catchUp?: boolean; + /** Retention window in days, or null to keep everything. Omitted = server + * default (1 year). Bounds the fetch AND the prune together. */ + windowDays?: number | null; /** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */ slot?: number; } @@ -96,6 +99,7 @@ export async function requestIndex(options: IndexRequestOptions = {}): Promise { - return requestIndex({ catchUp: true, slot }); +export async function catchUpIndex(slot?: number, windowDays?: number | null): Promise { + return requestIndex({ catchUp: true, slot, windowDays }); +} + +// ── Retention setting ──────────────────────────────────────────────────── +// Renderer-owned: the renderer already drives every index run, so keeping the +// choice here avoids a second source of truth on the server that could drift +// out of step with what the user last picked. + +const RETENTION_KEY = 'vncmail:index:retention-days'; +/** Matches DEFAULT_RETENTION_WINDOW_DAYS in lib/mail-index/reindex.ts. */ +export const DEFAULT_RETENTION_DAYS = 365; + +/** `null` = keep everything. */ +export function getRetentionDays(): number | null { + if (typeof window === 'undefined') return DEFAULT_RETENTION_DAYS; + const raw = window.localStorage.getItem(RETENTION_KEY); + if (raw === null) return DEFAULT_RETENTION_DAYS; + if (raw === 'forever') return null; + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) ? parsed : DEFAULT_RETENTION_DAYS; +} + +export function setRetentionDays(days: number | null): void { + if (typeof window === 'undefined') return; + window.localStorage.setItem(RETENTION_KEY, days === null ? 'forever' : String(days)); } export interface IndexStats { diff --git a/lib/mail-index/__tests__/recency.test.ts b/lib/mail-index/__tests__/recency.test.ts new file mode 100644 index 00000000..0b5d4016 --- /dev/null +++ b/lib/mail-index/__tests__/recency.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest'; +import { detectRecencyIntent } from '../recency'; + +// Fixed "now" so the month-name branch is deterministic: 2026-08-07. +const NOW = new Date('2026-08-07T10:00:00.000Z'); + +describe('detectRecencyIntent', () => { + it('recognises the two questions that actually failed against a populated index', () => { + // Both were asked by a real user and both returned nothing useful, because + // bm25 matched the WORDS "last"/"July" rather than the dates. + expect(detectRecencyIntent('who sent the last email', NOW)).not.toBeNull(); + expect(detectRecencyIntent('summarize the input of all mails sent in July', NOW)).not.toBeNull(); + }); + + it('turns a month name into a bounded range, not a top-N', () => { + const intent = detectRecencyIntent('summarize all mails sent in July', NOW); + // Asserted in LOCAL time on purpose. The bounds are local midnight + // rendered as UTC instants, so west of UTC the ISO string reads as the + // previous month - and that is correct, not a bug: a mail that arrived + // 00:30 local on 1 July belongs to the user's July even though its UTC + // timestamp says 30 June. Asserting the ISO prefix would enshrine the + // wrong semantics and pass only in UTC. + const since = new Date(intent!.since!); + const until = new Date(intent!.until!); + expect(since.getMonth()).toBe(6); // local July... + expect(since.getDate()).toBe(1); // ...starting on the 1st + expect(until.getMonth()).toBe(7); // exclusive upper bound = local 1 Aug + expect(until.getDate()).toBe(1); + }); + + it('reads a month later in the year as LAST year', () => { + // Asked in August, "December" cannot mean four months from now. + const intent = detectRecencyIntent('what came in December?', NOW); + const since = new Date(intent!.since!); + expect(since.getFullYear()).toBe(2025); + expect(since.getMonth()).toBe(11); + }); + + it('handles today and yesterday as distinct bounded days', () => { + const today = detectRecencyIntent('anything today?', NOW); + expect(today?.since).toBeDefined(); + expect(today?.until).toBeUndefined(); + + const yesterday = detectRecencyIntent('what arrived yesterday', NOW); + expect(yesterday?.since).toBeDefined(); + expect(yesterday?.until).toBeDefined(); + expect(new Date(yesterday!.until!).getTime()).toBeGreaterThan(new Date(yesterday!.since!).getTime()); + }); + + it('understands German recency wording — the app ships a German UI', () => { + expect(detectRecencyIntent('welche war die letzte Mail?', NOW)).not.toBeNull(); + expect(detectRecencyIntent('was kam heute an', NOW)).not.toBeNull(); + const juli = detectRecencyIntent('Mails aus Juli zusammenfassen', NOW); + expect(new Date(juli!.since!).getMonth()).toBe(6); + }); + + it('gives an unbounded top-N when recency is implied but no period named', () => { + const intent = detectRecencyIntent('what is the newest message', NOW); + expect(intent?.since).toBeUndefined(); + expect(intent?.until).toBeUndefined(); + expect(intent?.limit).toBeGreaterThan(0); + }); + + it('does NOT fire on pure content questions — those are keyword search\'s job', () => { + expect(detectRecencyIntent('what did Anna say about the invoice?', NOW)).toBeNull(); + expect(detectRecencyIntent('when is check-in for the Villa sul Lago booking?', NOW)).toBeNull(); + expect(detectRecencyIntent('find the contract with Bechtle', NOW)).toBeNull(); + }); + + it('does not treat a word merely CONTAINING a keyword as recency', () => { + // "newsletter" contains "new"; "lastly" contains "last". Word boundaries + // matter or half a mailbox reads as a time question. + expect(detectRecencyIntent('unsubscribe from the newsletter', NOW)).toBeNull(); + }); +}); diff --git a/lib/mail-index/jmap.ts b/lib/mail-index/jmap.ts index 35ae4acd..f12bf9d1 100644 --- a/lib/mail-index/jmap.ts +++ b/lib/mail-index/jmap.ts @@ -242,13 +242,17 @@ export async function queryRecentEmailIds( session: JmapSessionInfo, authHeader: string, accountId: string, - afterIso: string, + /** Lower bound, or undefined for "no date bound" (the keep-everything + * retention choice). An `after` of undefined must be OMITTED from the + * filter, not sent as undefined - Stalwart rejects a malformed filter + * rather than treating it as unset. */ + afterIso: string | undefined, limit: number, ): Promise { const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [ ['Email/query', { accountId, - filter: { after: afterIso }, + filter: afterIso ? { after: afterIso } : {}, sort: [{ property: 'receivedAt', isAscending: false }], limit, calculateTotal: false, diff --git a/lib/mail-index/recency.ts b/lib/mail-index/recency.ts new file mode 100644 index 00000000..88f354da --- /dev/null +++ b/lib/mail-index/recency.ts @@ -0,0 +1,92 @@ +// Recency-intent detection for the retrieval layer. +// +// Keyword search cannot answer a question about WHEN. bm25 ranks by term +// overlap, so "what was the last mail I received" matches documents that +// happen to contain the word "last", and "summarise everything from July" +// matches documents containing "July" — not documents dated in July. Both were +// asked by a real user against a correctly-populated index and both returned +// nothing useful, which is what this module exists to fix: it decides when a +// question is really a date question, and turns it into a date RANGE the index +// can answer with an ordered scan over `occurred_at`. +// +// Deliberately a heuristic on English/German keywords rather than an LLM call: +// it runs on every question, must be instant, and a false positive is cheap +// (the recency hits are fused with the keyword hits, not substituted for them). + +// TIMEZONE NOTE: all bounds are built from LOCAL calendar boundaries and then +// serialised as UTC instants. That is deliberate — "July" means the user's +// July, so a mail received 00:30 local on 1 July belongs to it even though its +// stored UTC timestamp reads 30 June. Building the bounds in UTC instead would +// silently drop the first/last hours of every named period for anyone not on +// UTC. +export interface RecencyIntent { + /** ISO lower bound, if the question named one. */ + since?: string; + /** ISO upper bound, if the question named a closed period. */ + until?: string; + /** How many documents the recency leg should contribute. */ + limit: number; +} + +const RECENCY_WORDS = [ + // English + 'last', 'latest', 'recent', 'recently', 'newest', 'new', 'today', 'yesterday', + 'this week', 'this month', 'past week', 'past month', 'so far', 'just now', 'current', + // German — the app ships a German UI and users mix languages freely + 'letzte', 'letzten', 'letzter', 'neueste', 'neuesten', 'neu', 'heute', 'gestern', + 'diese woche', 'diesen monat', 'kürzlich', 'zuletzt', 'aktuell', +]; + +const MONTHS: Record = { + january: 0, february: 1, march: 2, april: 3, may: 4, june: 5, + july: 6, august: 7, september: 8, october: 9, november: 10, december: 11, + januar: 0, februar: 1, märz: 2, maerz: 2, mai: 4, juni: 5, + juli: 6, oktober: 9, dezember: 11, +}; + +function startOfDay(d: Date): Date { + const c = new Date(d); + c.setHours(0, 0, 0, 0); + return c; +} + +/** + * @param now injected so the behaviour is testable and deterministic — the + * month-name branch depends on "which year is it" and must not be a coin + * flip in a test suite. + */ +export function detectRecencyIntent(question: string, now: Date = new Date()): RecencyIntent | null { + const q = question.toLowerCase(); + + // A named month wins over generic recency words: "everything from July" is a + // bounded range, which is far more useful than "the newest N". + for (const [name, monthIndex] of Object.entries(MONTHS)) { + if (!new RegExp(`\\b${name}\\b`).test(q)) continue; + // A month later than the current one must mean LAST year — "July" asked in + // March means the July that already happened, not one nine months away. + const year = monthIndex > now.getMonth() ? now.getFullYear() - 1 : now.getFullYear(); + const since = new Date(year, monthIndex, 1, 0, 0, 0, 0); + const until = new Date(year, monthIndex + 1, 1, 0, 0, 0, 0); + return { since: since.toISOString(), until: until.toISOString(), limit: 40 }; + } + + if (/\btoday\b|\bheute\b/.test(q)) { + return { since: startOfDay(now).toISOString(), limit: 25 }; + } + if (/\byesterday\b|\bgestern\b/.test(q)) { + const start = startOfDay(new Date(now.getTime() - 86_400_000)); + return { since: start.toISOString(), until: startOfDay(now).toISOString(), limit: 25 }; + } + if (/this week|past week|diese woche|letzte woche/.test(q)) { + return { since: startOfDay(new Date(now.getTime() - 7 * 86_400_000)).toISOString(), limit: 40 }; + } + if (/this month|past month|diesen monat|letzten monat/.test(q)) { + return { since: startOfDay(new Date(now.getTime() - 30 * 86_400_000)).toISOString(), limit: 40 }; + } + + if (RECENCY_WORDS.some((w) => (w.includes(' ') ? q.includes(w) : new RegExp(`\\b${w}\\b`).test(q)))) { + // No period named — "the last mail", "what's new". Unbounded top-N. + return { limit: 15 }; + } + return null; +} diff --git a/lib/mail-index/reindex.ts b/lib/mail-index/reindex.ts index ac30ab4c..9a177e55 100644 --- a/lib/mail-index/reindex.ts +++ b/lib/mail-index/reindex.ts @@ -29,16 +29,51 @@ import { withIndexKey } from './key'; import { getStoreDir } from './paths'; import { MailIndex, type ContentType, type IndexDoc } from './store'; -/** - * Bounded window. Small on purpose: this is the first cut of a retrieval index, - * and a wide window turns "index on every delivery" into a slow request. The - * event-driven path indexes single objects, so the window only bounds catch-up. - */ -export const INDEX_WINDOW_DAYS = 30; /** Calendar looks forward as well as back - upcoming events are the useful ones. */ export const CALENDAR_FORWARD_DAYS = 180; -/** Per-type ceiling for one catch-up pass. */ +/** Per-type ceiling for one catch-up pass, per 30 days of window. */ export const CATCHUP_MAX_PER_TYPE = 500; + +/** + * How much mail history the local index keeps. User-selectable + * (Settings -> About & Data); `null` means keep everything and never prune. + * + * This is NOT just a fetch bound - catch-up also PRUNES mail older than it. + * The original hardcoded 30 days therefore made a question like "summarise + * everything from July" unanswerable in August: the rows had been deliberately + * deleted, while the UI said only that nothing matched. A real user hit exactly + * that, which is why this is a setting with a year-long default rather than a + * constant tuned for a first cut. + */ +export const RETENTION_CHOICES = [30, 90, 365, null] as const; +export type RetentionWindowDays = (typeof RETENTION_CHOICES)[number]; +export const DEFAULT_RETENTION_WINDOW_DAYS: RetentionWindowDays = 365; + +/** Hard ceiling regardless of window - one pass must still terminate. */ +export const CATCHUP_MAX_PER_TYPE_UNLIMITED = 20_000; + +export function normalizeWindowDays(raw: unknown): RetentionWindowDays { + if (raw === null) return null; + if (typeof raw !== 'number' || !Number.isFinite(raw)) return DEFAULT_RETENTION_WINDOW_DAYS; + // Anything off the list falls back to the default rather than being honoured + // verbatim - this value drives DELETION, so a typo'd 0 must never silently + // wipe the index. + const allowed: readonly number[] = RETENTION_CHOICES.filter((c) => c !== null); + return allowed.includes(raw) ? (raw as RetentionWindowDays) : DEFAULT_RETENTION_WINDOW_DAYS; +} + +/** Scales the per-pass ceiling with the window: 500 is right for a month and + * nonsense for "everything", where having the history IS the point. */ +export function catchUpCapFor(windowDays: RetentionWindowDays): number { + if (windowDays === null) return CATCHUP_MAX_PER_TYPE_UNLIMITED; + return Math.min(CATCHUP_MAX_PER_TYPE_UNLIMITED, Math.round((windowDays / 30) * CATCHUP_MAX_PER_TYPE)); +} + +/** Lower bound for a date-filtered query, or undefined when unlimited. */ +export function windowStartIso(windowDays: RetentionWindowDays): string | undefined { + return windowDays === null ? undefined : isoDaysFromNow(-windowDays); +} + /** Ids accepted in one event-driven call. A push reports a handful, not thousands. */ export const MAX_IDS_PER_CALL = 200; /** Cap on body bytes requested per message from the server. */ @@ -169,6 +204,7 @@ interface FetchArgs { authHeader: string; jmapAccountId: string; ids: readonly string[] | null; + windowDays: RetentionWindowDays; } interface FetchResult { @@ -184,13 +220,14 @@ interface FetchResult { /** Fetches and flattens one content type. `ids === null` means "the recent window". */ async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise { - const { session, authHeader, jmapAccountId, ids } = args; + const { session, authHeader, jmapAccountId, ids, windowDays } = args; + const cap = catchUpCapFor(windowDays); switch (contentType) { case 'mail': { const targetIds = ids ?? await queryRecentEmailIds( session, authHeader, jmapAccountId, - isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE, + windowStartIso(windowDays), cap, ); const docs: IndexDoc[] = []; // Chunked because bodies are big: one Email/get for 500 messages with @@ -206,8 +243,11 @@ async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise>; /** Drop documents outside the retention window after writing. */ prune?: boolean; + /** Retention window in days, or null to keep everything. Bounds BOTH the + * catch-up fetch and the prune, so the two can never disagree and delete + * what was just written. Defaults to DEFAULT_RETENTION_WINDOW_DAYS. */ + windowDays?: RetentionWindowDays; } /** @@ -274,6 +318,7 @@ export async function runIndex( req: IndexRequest, ): Promise { const started = Date.now(); + const windowDays = normalizeWindowDays(req.windowDays === undefined ? DEFAULT_RETENTION_WINDOW_DAYS : req.windowDays); const storeDir = getStoreDir(); if (!storeDir) { throw new IndexSessionError('The local index is not enabled in this deployment.', 404); @@ -325,14 +370,18 @@ export async function runIndex( : null; const { docs, queriedCount } = await fetchDocs(contentType, { - session, authHeader: indexSession.authHeader, jmapAccountId, ids, + session, authHeader: indexSession.authHeader, jmapAccountId, ids, windowDays, }); written[contentType] = index.upsert(docs); if (req.prune && contentType === 'mail') { // Only mail prunes by date: calendar's window looks forward, // contacts have no date, and file rows are metadata-sized. - index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS)); + // A null window means keep everything - pruning is SKIPPED, not + // run with some fallback bound, or the setting would silently + // delete the history the user just asked to retain. + const cutoff = windowStartIso(windowDays); + if (cutoff) index.pruneOlderThan(jmapAccountId, 'mail', cutoff); } // Contact/file DELETES: a JMAP `destroyed` only ever reaches this diff --git a/lib/mail-index/store.ts b/lib/mail-index/store.ts index dddbb3a1..13735219 100644 --- a/lib/mail-index/store.ts +++ b/lib/mail-index/store.ts @@ -374,6 +374,69 @@ export class MailIndex { })); } + /** + * Newest documents by date, ignoring keyword relevance entirely. + * + * The retrieval leg for RECENCY questions — "what was the last mail", "who + * wrote most recently", "everything from July". Full-text search cannot + * answer those even with a perfect index: bm25 ranks by term overlap and has + * no notion of "latest", so "the last mail" matches documents containing the + * word "last". Two real questions failed exactly that way before this + * existed. `doc(jmap_account_id, content_type, occurred_at DESC)` is already + * indexed, so this is an ordered range scan, not a table sweep. + * + * `since`/`until` are ISO strings, both optional — a month-name question + * becomes a bounded range, a bare "latest" becomes an unbounded top-N. + */ + recent(opts: { + types?: readonly ContentType[]; + limit?: number; + since?: string; + until?: string; + snippetChars?: number; + }): SearchHit[] { + const limit = Math.min(Math.max(opts.limit ?? 10, 1), 200); + const snippetChars = Math.min(Math.max(opts.snippetChars ?? 400, 80), 2000); + const types = opts.types && opts.types.length > 0 ? opts.types : null; + + const where: string[] = ['d.occurred_at IS NOT NULL']; + const params: Array = []; + if (types) { + where.push(`d.content_type IN (${types.map(() => '?').join(',')})`); + params.push(...types); + } + if (opts.since) { where.push('d.occurred_at >= ?'); params.push(opts.since); } + if (opts.until) { where.push('d.occurred_at <= ?'); params.push(opts.until); } + + const rows = this.db + .prepare(` + SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people, + d.occurred_at, d.metadata_json, + substr(f.body, 1, ${snippetChars}) AS snip + FROM doc d + LEFT JOIN doc_fts f ON f.rowid = d.rowid + WHERE ${where.join(' AND ')} + ORDER BY d.occurred_at DESC + LIMIT ? + `) + .all([...params, limit]); + + return rows.map((r) => ({ + contentType: String(r.content_type) as ContentType, + id: String(r.id), + jmapAccountId: String(r.jmap_account_id), + title: String(r.title ?? ''), + people: String(r.people ?? ''), + occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at), + metadata: safeParseObject(r.metadata_json), + // No bm25 score here: these are ordered by time, not relevance, and + // faking a relevance number would let the fusion step rank them as if + // they had been scored. + score: 0, + snippet: String(r.snip ?? ''), + })); + } + /** Per-type counts and freshness, for the Settings UI and for debugging. */ stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> { return this.db From 1a19e96bcbb7c8ed8d5ca2fdc0727c00dfee0c8f Mon Sep 17 00:00:00 2001 From: vncmail-ci Date: Fri, 7 Aug 2026 07:48:51 +0000 Subject: [PATCH 3/6] chore(deploy): pin dev to sha-6dc6ad09 [skip ci] --- deploy/k8s/overlays/dev/image-tag/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml index 8770c469..ec89bc68 100644 --- a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml +++ b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml @@ -5,4 +5,4 @@ kind: Component images: - name: vncmail-plus newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus - newTag: sha-f6fc34fa + newTag: sha-6dc6ad09 From bd778adf123995968e7070efdfff2a46504be84b Mon Sep 17 00:00:00 2001 From: Bernd Rodler Date: Fri, 7 Aug 2026 09:57:51 +0200 Subject: [PATCH 4/6] feat(electron): supervise a password-protected opencode server (B1+B3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B1 — LIFECYCLE. The OpenCode class previously required the user to remember to run `opencode serve` in a terminal before opening their mail app, and again after every reboot; in practice that means the feature quietly stops existing. The desktop shell now owns it: finds the binary (OPENCODE_BIN, then ~/.opencode/bin — its installer's default, which is NOT on the PATH a macOS GUI app inherits, so PATH alone finds nothing for most users), starts it on a free port, restarts up to 3 times if it dies, and kills it on quit. Absent binary = the class simply stays unavailable, no error. B3 — SECURITY. opencode's own startup warns "OPENCODE_SERVER_PASSWORD is not set; server is unsecured" — without one, any local process can drive the agent. A per-launch password is now always generated (never persisted: the server dies with the app, so a durable secret would be pure liability) and handed to the standalone server alongside the base URL. The auth scheme is worth recording because it is NOT in opencode's own OpenAPI spec, which declares no securitySchemes at all: HTTP Basic with the username EXACTLY `opencode`. Verified against 1.18.14 by trying them — an empty username, an arbitrary one, Bearer, and every plausible custom header all 401 with the correct password. Pinned by a unit test that decodes the header, so a future refactor can't silently drop it. Verified live against a real password-protected server on 4097: authenticated discovery + prompt round-tripped, AND the same call with no password was rejected — proving the auth is real rather than decorative. Also removed now-stale guidance: the 503 no longer says "start one with opencode serve", because the app does that; it says to install the CLI. Gate: tsc clean, eslint clean, build clean, 2521/2522 tests. The one failure is lib/__tests__/jmap-client-resilience.test.ts's onConnectionChange timing flake — byte-identical to what is already running in prod (git diff vs origin/main for that file and lib/jmap/ is empty), pre-existing, and unrelated to anything here. --- app/api/ai/opencode/chat/route.ts | 2 +- app/api/ai/opencode/models/route.ts | 2 +- electron/main.ts | 91 +++++++++++++++++++++++++++++ lib/ai/__tests__/opencode.test.ts | 35 +++++++++++ lib/ai/opencode.ts | 22 ++++++- 5 files changed, 149 insertions(+), 3 deletions(-) diff --git a/app/api/ai/opencode/chat/route.ts b/app/api/ai/opencode/chat/route.ts index 5c97ac63..bde99901 100644 --- a/app/api/ai/opencode/chat/route.ts +++ b/app/api/ai/opencode/chat/route.ts @@ -54,7 +54,7 @@ export async function POST(request: NextRequest) { const found = await findOpencodeServer(); if (!found) { return NextResponse.json( - { error: 'No local OpenCode server found. Start one with: opencode serve --port 4096' }, + { error: 'No local OpenCode server is running. The desktop app starts one automatically when the opencode CLI is installed \u2014 install it from opencode.ai, then restart VNCmail+.' }, { status: 503 }, ); } diff --git a/app/api/ai/opencode/models/route.ts b/app/api/ai/opencode/models/route.ts index 7b5e668a..ae68b4e1 100644 --- a/app/api/ai/opencode/models/route.ts +++ b/app/api/ai/opencode/models/route.ts @@ -31,7 +31,7 @@ export async function GET(request: NextRequest) { // isn't running), and the client turns it into setup guidance rather than // an error banner. return NextResponse.json( - { error: 'No local OpenCode server found. Start one with: opencode serve --port 4096' }, + { error: 'No local OpenCode server is running. The desktop app starts one automatically when the opencode CLI is installed \u2014 install it from opencode.ai, then restart VNCmail+.' }, { status: 503 }, ); } diff --git a/electron/main.ts b/electron/main.ts index 855a6403..91fda357 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -160,6 +160,89 @@ function ensureSessionSecretFile(): string | null { } } +// ── OpenCode agent server ──────────────────────────────────────────────── +// +// The `opencode` AI class talks to a locally-running `opencode serve`. Left to +// the user that means "remember to start a terminal process before opening +// your mail app, and again after every reboot" - which is to say the feature +// quietly stops existing. So the desktop shell owns its lifecycle: start it if +// the binary is installed, restart it if it dies, kill it on quit. +// +// SECURITY: opencode itself warns "OPENCODE_SERVER_PASSWORD is not set; server +// is unsecured" - without one, any local process can drive the agent. We always +// generate one. Auth is HTTP Basic with the username EXACTLY `opencode` +// (verified against 1.18.14: an empty or arbitrary username 401s even with the +// right password, and no bearer/custom-header form works) - undocumented in its +// own OpenAPI spec, which declares no securitySchemes at all. + +let opencodeProcess: ChildProcess | null = null; +let opencodeRestarts = 0; +/** Set by stopOpencodeServer() so the exit handler can tell a deliberate + * shutdown from a crash and not fight the quit by respawning. */ +let opencodeStopping = false; +const OPENCODE_MAX_RESTARTS = 3; + +/** Where the binary lives. `~/.opencode/bin` is its own installer's default and + * is NOT on the PATH a GUI app inherits on macOS, so PATH alone finds nothing + * for most users. */ +function findOpencodeBinary(): string | null { + const explicit = process.env.OPENCODE_BIN?.trim(); + if (explicit && fs.existsSync(explicit)) return explicit; + const candidates = [ + path.join(app.getPath("home"), ".opencode", "bin", "opencode"), + "/opt/homebrew/bin/opencode", + "/usr/local/bin/opencode", + "/usr/bin/opencode", + ]; + return candidates.find((c) => fs.existsSync(c)) ?? null; +} + +interface OpencodeHandle { + baseUrl: string; + password: string; +} + +async function startOpencodeServer(): Promise { + const binary = findOpencodeBinary(); + if (!binary) return null; // not installed - the class simply stays unavailable + + const port = await getFreePort(); + // Per-launch, never persisted: the server dies with the app, so there is no + // value in a durable secret and every reason not to leave one on disk. + const password = randomBytes(24).toString("hex"); + const baseUrl = `http://127.0.0.1:${port}`; + + const spawnOnce = () => { + opencodeProcess = spawn(binary, ["serve", "--port", String(port), "--hostname", "127.0.0.1"], { + env: { ...process.env, OPENCODE_SERVER_PASSWORD: password }, + stdio: "ignore", + }); + opencodeProcess.on("exit", (code, signal) => { + opencodeProcess = null; + // A deliberate shutdown arrives as SIGTERM from stopOpencodeServer(). + if (opencodeStopping || signal === "SIGTERM") return; + if (opencodeRestarts >= OPENCODE_MAX_RESTARTS) { + console.error(`[opencode] gave up restarting after ${OPENCODE_MAX_RESTARTS} attempts (last code=${code})`); + return; + } + opencodeRestarts += 1; + console.error(`[opencode] server exited (code=${code}); restart ${opencodeRestarts}/${OPENCODE_MAX_RESTARTS}`); + setTimeout(spawnOnce, 1000 * opencodeRestarts); + }); + }; + spawnOnce(); + + return { baseUrl, password }; +} + +function stopOpencodeServer(): void { + opencodeStopping = true; + if (!opencodeProcess) return; + const proc = opencodeProcess; + opencodeProcess = null; + proc.kill("SIGTERM"); +} + /** * Locates the standalone server's entrypoint. Packaged builds ship it as an * extraResource (see electron-builder.config.js) because .next/standalone @@ -256,6 +339,9 @@ async function startStandaloneServer(): Promise { // using the OS keychain at all. The fd NUMBER below is not a secret; only // what travels over it is. const sessionSecretFile = ensureSessionSecretFile(); + // Started before the app server so its address can be handed over as env; + // null when opencode isn't installed, in which case the class stays absent. + const opencode = await startOpencodeServer(); serverProcess = spawn(process.execPath, [serverEntry], { env: { @@ -268,6 +354,9 @@ async function startStandaloneServer(): Promise { // SESSION_SECRET env var outranks any file in getSessionSecret()'s // resolution order regardless). ...(sessionSecretFile ? { SESSION_SECRET_FILE: sessionSecretFile } : {}), + ...(opencode + ? { OPENCODE_BASE_URL: opencode.baseUrl, OPENCODE_SERVER_PASSWORD: opencode.password } + : {}), ...process.env, ELECTRON_RUN_AS_NODE: "1", PORT: String(port), @@ -408,6 +497,7 @@ app.whenReady().then(() => { app.on("window-all-closed", () => { stopStandaloneServer(); + stopOpencodeServer(); if (process.platform !== "darwin") { app.quit(); } @@ -415,6 +505,7 @@ app.on("window-all-closed", () => { app.on("before-quit", () => { stopStandaloneServer(); + stopOpencodeServer(); }); app.on("activate", () => { diff --git a/lib/ai/__tests__/opencode.test.ts b/lib/ai/__tests__/opencode.test.ts index 71355ce8..4ac676a0 100644 --- a/lib/ai/__tests__/opencode.test.ts +++ b/lib/ai/__tests__/opencode.test.ts @@ -68,6 +68,41 @@ describe('opencodeBaseUrls', () => { }); }); +describe('auth', () => { + const originalFetch = global.fetch; + const originalPw = process.env.OPENCODE_SERVER_PASSWORD; + afterEach(() => { + global.fetch = originalFetch; + if (originalPw === undefined) delete process.env.OPENCODE_SERVER_PASSWORD; + else process.env.OPENCODE_SERVER_PASSWORD = originalPw; + vi.restoreAllMocks(); + }); + + it('sends HTTP Basic with the username EXACTLY "opencode"', async () => { + // Verified against 1.18.14: an empty or arbitrary username 401s even with + // the right password, and no bearer/custom-header form works. Its OpenAPI + // spec declares no securitySchemes, so this is only knowable by trying it + // - which makes it exactly the kind of thing to pin with a test. + process.env.OPENCODE_SERVER_PASSWORD = 'hunter2'; + const fetchMock = vi.fn().mockResolvedValue(jsonResponse({ data: [{ id: 'm', providerID: 'p' }] })); + global.fetch = fetchMock as unknown as typeof fetch; + + await findOpencodeServer(); + const sentHeaders = fetchMock.mock.calls[0][1].headers as Record; + const decoded = Buffer.from(sentHeaders.Authorization.replace('Basic ', ''), 'base64').toString(); + expect(decoded).toBe('opencode:hunter2'); + }); + + it('sends no auth header at all when no password is configured', async () => { + delete process.env.OPENCODE_SERVER_PASSWORD; + const fetchMock = vi.fn().mockResolvedValue(jsonResponse({ data: [{ id: 'm', providerID: 'p' }] })); + global.fetch = fetchMock as unknown as typeof fetch; + await findOpencodeServer(); + const sentHeaders = fetchMock.mock.calls[0][1].headers as Record; + expect(sentHeaders.Authorization).toBeUndefined(); + }); +}); + describe('findOpencodeServer', () => { const originalFetch = global.fetch; afterEach(() => { diff --git a/lib/ai/opencode.ts b/lib/ai/opencode.ts index e25c3887..1546f1b9 100644 --- a/lib/ai/opencode.ts +++ b/lib/ai/opencode.ts @@ -73,11 +73,31 @@ export function parseModelRef(ref: string): { providerID: string; modelID: strin return { providerID: ref.slice(0, slash), modelID: ref.slice(slash + 1) }; } +/** + * Auth header for a password-protected server. + * + * HTTP Basic with the username EXACTLY `opencode` — verified against 1.18.14: + * an empty username, an arbitrary one, a Bearer token and every plausible + * custom header all 401 with the correct password. Its own OpenAPI spec + * declares no securitySchemes at all, so this is only knowable by trying it. + * Absent password = an unsecured server (the desktop shell always sets one; + * a hand-started `opencode serve` typically has none). + */ +function authHeaders(): Record { + const password = process.env.OPENCODE_SERVER_PASSWORD; + if (!password) return {}; + return { Authorization: `Basic ${Buffer.from(`opencode:${password}`).toString('base64')}` }; +} + async function fetchJson(url: string, init: RequestInit, timeoutMs: number): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { - const res = await fetch(url, { ...init, signal: controller.signal }); + const res = await fetch(url, { + ...init, + headers: { ...authHeaders(), ...(init.headers as Record | undefined) }, + signal: controller.signal, + }); if (!res.ok) return null; // The SPA catch-all returns HTML with a 200 for unknown paths — see the // module header. Content-type is what actually distinguishes a real API From 372722a903912a905e76415de8aea2b0f54d4413 Mon Sep 17 00:00:00 2001 From: vncmail-ci Date: Fri, 7 Aug 2026 08:01:28 +0000 Subject: [PATCH 5/6] chore(deploy): pin dev to sha-35ed6a28 [skip ci] --- deploy/k8s/overlays/dev/image-tag/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml index ec89bc68..8e42e479 100644 --- a/deploy/k8s/overlays/dev/image-tag/kustomization.yaml +++ b/deploy/k8s/overlays/dev/image-tag/kustomization.yaml @@ -5,4 +5,4 @@ kind: Component images: - name: vncmail-plus newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus - newTag: sha-6dc6ad09 + newTag: sha-35ed6a28 From 62b0455388fdf5d5c36bc41a07dc66bd8dcd528b Mon Sep 17 00:00:00 2001 From: Bernd Rodler Date: Fri, 7 Aug 2026 12:09:14 +0200 Subject: [PATCH 6/6] =?UTF-8?q?feat(ai):=20OpenCode=20provider=20managemen?= =?UTF-8?q?t=20(B4)=20=E2=80=94=20"any=20LLM=20OpenCode=20supports",=20fro?= =?UTF-8?q?m=20inside=20VNCmail+?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this, the OpenCode class could only use providers already authenticated via its own CLI (opencode auth login) — this app could pick a MODEL, never add a PROVIDER. That is the one thing standing between "OpenCode integration" and the actual ask: any LLM it supports, added from here. New GET/PUT/DELETE /api/ai/opencode/providers, backed by GET /provider (every provider OpenCode knows — 180 on a real run) and GET /provider/auth (which auth method each accepts). New "Manage providers" panel in the OpenCode settings section: search, add a key, remove one. Scoped to API-key auth only, deliberately — recorded in lib/ai/opencode.ts's module comment. `PUT /auth/{id}` with `{type:'api', key}` is one HTTP call with a schema-verified shape. OAuth entries in /provider/auth need a browser redirect + callback this app has no page for, and some carry interactive prompts beyond a single form (GitHub Copilot's deployment-type picker) — real scope for later, not something to half-build. OAuth-only providers are still LISTED, just marked "Browser sign-in only" rather than hidden, so the picker stays honest about what it can't do here. A real finding from testing this against opencode's actual behaviour rather than trusting a 200: NOT EVERY PROVIDER BECOMES CONNECTED FROM A BARE API KEY. Snowflake Cortex needs SNOWFLAKE_ACCOUNT alongside its token; a single key field silently leaves it stored-but-unconnected with no error from the PUT itself. Worse, the provider's own `env` array length does not predict this — Azure also needs two env vars and DOES connect from one key. There is no reliable way to know in advance, so the route now VERIFIES by re-listing providers after the write and reports plainly when a key was accepted but the provider still isn't connected, rather than reporting the PUT's own success. Verified live end-to-end, twice: once confirming a simple single-field provider connects and can be removed cleanly, once confirming the honest "stored but not connected" case is real and detected, not theoretical. Cleaned up every throwaway credential from this machine's real opencode config afterwards (checked auth.json directly, not just this app's view of it). 17 new/updated unit tests. Gate: tsc clean, eslint clean, 2527/2527 tests, build clean. --- app/api/ai/opencode/providers/route.ts | 103 ++++++++++++ components/settings/ai-assistant-settings.tsx | 159 ++++++++++++++++++ lib/ai/__tests__/opencode.test.ts | 86 ++++++++++ lib/ai/local-client.ts | 39 +++++ lib/ai/opencode.ts | 73 ++++++++ 5 files changed, 460 insertions(+) create mode 100644 app/api/ai/opencode/providers/route.ts diff --git a/app/api/ai/opencode/providers/route.ts b/app/api/ai/opencode/providers/route.ts new file mode 100644 index 00000000..88014b8f --- /dev/null +++ b/app/api/ai/opencode/providers/route.ts @@ -0,0 +1,103 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getStalwartCredentials } from '@/lib/stalwart/credentials'; +import { configManager } from '@/lib/admin/config-manager'; +import { + findOpencodeServer, listOpencodeProviders, setOpencodeProviderKey, removeOpencodeProvider, +} from '@/lib/ai/opencode'; +import { logger } from '@/lib/logger'; + +export const runtime = 'nodejs'; + +const SETUP_ERROR = + 'No local OpenCode server is running. The desktop app starts one automatically when the opencode CLI is installed — install it from opencode.ai, then restart VNCmail+.'; + +async function requireOpencode(request: NextRequest) { + const auth = await getStalwartCredentials(request); + if (!auth) return { error: NextResponse.json({ error: 'not authenticated' }, { status: 401 }) } as const; + + await configManager.ensureLoaded(); + if (configManager.getAiConsoleConfig().classesEnabled.opencode === false) { + return { error: NextResponse.json({ error: 'the OpenCode class is disabled by admin policy' }, { status: 403 }) } as const; + } + + const found = await findOpencodeServer(); + if (!found) return { error: NextResponse.json({ error: SETUP_ERROR }, { status: 503 }) } as const; + return { baseUrl: found.baseUrl } as const; +} + +/** + * GET/PUT/DELETE /api/ai/opencode/providers — lets a user add "any LLM + * OpenCode supports" from inside this app, rather than only whatever was + * already authenticated via its own CLI. See lib/ai/opencode.ts's module + * note on why this only covers API-key providers for now, not OAuth ones. + */ +export async function GET(request: NextRequest) { + const result = await requireOpencode(request); + if ('error' in result) return result.error; + try { + const providers = await listOpencodeProviders(result.baseUrl); + return NextResponse.json({ providers }, { headers: { 'Cache-Control': 'no-store' } }); + } catch (cause) { + logger.error('opencode providers list failed', { error: cause instanceof Error ? cause.message : String(cause) }); + return NextResponse.json({ error: 'Could not list OpenCode providers' }, { status: 502 }); + } +} + +export async function PUT(request: NextRequest) { + const result = await requireOpencode(request); + if ('error' in result) return result.error; + + let body: { providerID?: unknown; key?: unknown }; + try { + body = await request.json(); + } catch { + return NextResponse.json({ error: 'invalid JSON body' }, { status: 400 }); + } + const providerID = typeof body.providerID === 'string' ? body.providerID.trim() : ''; + const key = typeof body.key === 'string' ? body.key.trim() : ''; + if (!providerID || !key) { + return NextResponse.json({ error: 'providerID and key are required' }, { status: 400 }); + } + + try { + await setOpencodeProviderKey(result.baseUrl, providerID, key); + // VERIFY rather than trust the 200: OpenCode accepts a bare API key for + // every provider (confirmed live), but does not consider every provider + // "connected" from that alone - Snowflake Cortex, for one real example, + // needs SNOWFLAKE_ACCOUNT alongside its token, and a single key field + // silently leaves it unconnected with no error from the PUT itself. The + // provider's own `env` array length does NOT predict this reliably either + // (Azure needs two env vars and DOES connect from one key) - the only + // honest source of truth is asking OpenCode again. + const after = await listOpencodeProviders(result.baseUrl); + const nowConnected = after.find((p) => p.id === providerID)?.connected === true; + if (!nowConnected) { + return NextResponse.json({ + ok: false, + error: `OpenCode stored the key but does not show ${providerID} as connected — it likely needs more than one credential field (check its requirements with the opencode CLI: opencode auth login ${providerID}).`, + }, { status: 200 }); + } + return NextResponse.json({ ok: true }); + } catch (cause) { + logger.error('opencode provider auth failed', { providerID, error: cause instanceof Error ? cause.message : String(cause) }); + return NextResponse.json({ error: cause instanceof Error ? cause.message : 'Could not add the provider' }, { status: 502 }); + } +} + +export async function DELETE(request: NextRequest) { + const result = await requireOpencode(request); + if ('error' in result) return result.error; + + const providerID = request.nextUrl.searchParams.get('providerID')?.trim(); + if (!providerID) { + return NextResponse.json({ error: 'providerID is required' }, { status: 400 }); + } + + try { + await removeOpencodeProvider(result.baseUrl, providerID); + return NextResponse.json({ ok: true }); + } catch (cause) { + logger.error('opencode provider removal failed', { providerID, error: cause instanceof Error ? cause.message : String(cause) }); + return NextResponse.json({ error: cause instanceof Error ? cause.message : 'Could not remove the provider' }, { status: 502 }); + } +} diff --git a/components/settings/ai-assistant-settings.tsx b/components/settings/ai-assistant-settings.tsx index e4465ecb..d5fc3c8d 100644 --- a/components/settings/ai-assistant-settings.tsx +++ b/components/settings/ai-assistant-settings.tsx @@ -23,6 +23,10 @@ import { listLocalModels, listServerModels, listOpencodeModels, + listOpencodeProviders, + addOpencodeProvider, + removeOpencodeProvider, + type OpencodeProviderOption, type OpencodeModelOption, testLocalConnection, type AskResult, @@ -154,6 +158,58 @@ export function AiAssistantSettings() { const [refreshingOpencode, setRefreshingOpencode] = useState(false); const [opencodeError, setOpencodeError] = useState(null); + // ── OpenCode provider management — "add any LLM OpenCode supports" from + // inside this app, not only whatever its own CLI already authenticated. ── + const [opencodeProviders, setOpencodeProviders] = useState([]); + const [loadingProviders, setLoadingProviders] = useState(false); + const [providerSearch, setProviderSearch] = useState(''); + const [addingProviderId, setAddingProviderId] = useState(null); + const [newProviderKey, setNewProviderKey] = useState(''); + const [providerBusyId, setProviderBusyId] = useState(null); + const [providerActionError, setProviderActionError] = useState(null); + const [showProviderManager, setShowProviderManager] = useState(false); + + const refreshOpencodeProviders = useCallback(async () => { + setLoadingProviders(true); + setProviderActionError(null); + try { + setOpencodeProviders(await listOpencodeProviders()); + } catch (err) { + setProviderActionError(err instanceof Error ? err.message : String(err)); + } finally { + setLoadingProviders(false); + } + }, []); + + const handleAddProvider = useCallback(async (providerId: string) => { + if (!newProviderKey.trim()) return; + setProviderBusyId(providerId); + setProviderActionError(null); + try { + await addOpencodeProvider(providerId, newProviderKey.trim()); + setAddingProviderId(null); + setNewProviderKey(''); + await refreshOpencodeProviders(); + } catch (err) { + setProviderActionError(err instanceof Error ? err.message : String(err)); + } finally { + setProviderBusyId(null); + } + }, [newProviderKey, refreshOpencodeProviders]); + + const handleRemoveProvider = useCallback(async (providerId: string) => { + setProviderBusyId(providerId); + setProviderActionError(null); + try { + await removeOpencodeProvider(providerId); + await refreshOpencodeProviders(); + } catch (err) { + setProviderActionError(err instanceof Error ? err.message : String(err)); + } finally { + setProviderBusyId(null); + } + }, [refreshOpencodeProviders]); + const refreshOpencodeModels = useCallback(async () => { setRefreshingOpencode(true); setOpencodeError(null); @@ -423,6 +479,109 @@ export function AiAssistantSettings() { )} + + + + + + {showProviderManager && ( +
+ {providerActionError && ( +

+ {providerActionError} +

+ )} + +
+ setProviderSearch(e.target.value)} + placeholder="Search providers (e.g. anthropic, openai, groq)…" + spellCheck={false} + className={inputClass} + /> + +
+ + {opencodeProviders.length === 0 && !loadingProviders && ( +

No providers loaded yet — click Refresh.

+ )} + +
+ {opencodeProviders + .filter((p) => { + const q = providerSearch.trim().toLowerCase(); + return !q || p.id.toLowerCase().includes(q) || p.name.toLowerCase().includes(q); + }) + // Connected first (already sorted server-side), then cap what + // renders — 180 providers in one scroll box is noise, not choice. + .slice(0, providerSearch.trim() ? 40 : 20) + .map((p) => ( +
+
+ {p.name} + {p.id} +
+ {p.connected ? ( + <> + + Connected + + + + ) : p.supportsApiKey ? ( + addingProviderId === p.id ? ( +
+ setNewProviderKey(e.target.value)} + placeholder="API key" + autoFocus + className="px-2 py-1 text-xs rounded-md bg-muted border border-border w-36" + /> + + +
+ ) : ( + + ) + ) : ( + Browser sign-in only + )} +
+ ))} +
+
+ )} )} diff --git a/lib/ai/__tests__/opencode.test.ts b/lib/ai/__tests__/opencode.test.ts index 4ac676a0..4af77a54 100644 --- a/lib/ai/__tests__/opencode.test.ts +++ b/lib/ai/__tests__/opencode.test.ts @@ -180,3 +180,89 @@ describe('opencodePrompt', () => { expect(result).toEqual({ ok: false, error: 'OpenCode returned no message content' }); }); }); + +describe('listOpencodeProviders', () => { + const originalFetch = global.fetch; + afterEach(() => { + global.fetch = originalFetch; + vi.restoreAllMocks(); + }); + + it('merges /provider and /provider/auth into one list, connected first', async () => { + const fetchMock = vi.fn((url: string) => { + if (url.endsWith('/provider')) { + return Promise.resolve(jsonResponse({ + all: [{ id: 'anthropic', name: 'Anthropic' }, { id: 'deepseek', name: 'DeepSeek' }, { id: 'github-copilot', name: 'GitHub Copilot' }], + connected: ['deepseek'], + })); + } + if (url.endsWith('/provider/auth')) { + return Promise.resolve(jsonResponse({ + anthropic: [{ type: 'api' }], + deepseek: [{ type: 'api' }], + 'github-copilot': [{ type: 'oauth' }], + })); + } + return Promise.resolve(HTML_CATCHALL); + }); + global.fetch = fetchMock as unknown as typeof fetch; + + const { listOpencodeProviders } = await import('../opencode'); + const result = await listOpencodeProviders('http://127.0.0.1:4096'); + + expect(result).toHaveLength(3); + // Connected providers sort first regardless of name. + expect(result[0]).toMatchObject({ id: 'deepseek', connected: true, supportsApiKey: true }); + const anthropic = result.find((p) => p.id === 'anthropic'); + expect(anthropic).toMatchObject({ connected: false, supportsApiKey: true }); + const copilot = result.find((p) => p.id === 'github-copilot'); + // OAuth-only provider: listed, but honestly marked as not addable here. + expect(copilot).toMatchObject({ connected: false, supportsApiKey: false }); + }); + + it('returns an empty list rather than throwing when /provider is unreachable', async () => { + global.fetch = vi.fn().mockResolvedValue(HTML_CATCHALL) as unknown as typeof fetch; + const { listOpencodeProviders } = await import('../opencode'); + expect(await listOpencodeProviders('http://127.0.0.1:4096')).toEqual([]); + }); +}); + +describe('setOpencodeProviderKey / removeOpencodeProvider', () => { + const originalFetch = global.fetch; + afterEach(() => { + global.fetch = originalFetch; + vi.restoreAllMocks(); + }); + + it('PUTs the exact schema OpenCode requires: {type:"api", key}', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true }); + global.fetch = fetchMock as unknown as typeof fetch; + const { setOpencodeProviderKey } = await import('../opencode'); + + await setOpencodeProviderKey('http://127.0.0.1:4096', 'anthropic', 'sk-real-key'); + + expect(fetchMock).toHaveBeenCalledWith( + 'http://127.0.0.1:4096/auth/anthropic', + expect.objectContaining({ method: 'PUT', body: JSON.stringify({ type: 'api', key: 'sk-real-key' }) }), + ); + }); + + it('throws with the upstream status when OpenCode rejects the credential', async () => { + global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 400 }) as unknown as typeof fetch; + const { setOpencodeProviderKey } = await import('../opencode'); + await expect(setOpencodeProviderKey('http://127.0.0.1:4096', 'anthropic', 'bad')).rejects.toThrow(/400/); + }); + + it('DELETEs by provider id and encodes it in the path', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true }); + global.fetch = fetchMock as unknown as typeof fetch; + const { removeOpencodeProvider } = await import('../opencode'); + + await removeOpencodeProvider('http://127.0.0.1:4096', 'weird id/with slash'); + + expect(fetchMock).toHaveBeenCalledWith( + 'http://127.0.0.1:4096/auth/weird%20id%2Fwith%20slash', + expect.objectContaining({ method: 'DELETE' }), + ); + }); +}); diff --git a/lib/ai/local-client.ts b/lib/ai/local-client.ts index fd769053..37420720 100644 --- a/lib/ai/local-client.ts +++ b/lib/ai/local-client.ts @@ -181,6 +181,45 @@ export async function listOpencodeModels(): Promise { return (body?.models ?? []) as OpencodeModelOption[]; } +export interface OpencodeProviderOption { + id: string; + name: string; + connected: boolean; + supportsApiKey: boolean; +} + +/** Every provider OpenCode knows about, not just ones already authenticated — + * this is what lets "add any LLM OpenCode supports" mean something from + * inside this app instead of only whatever its CLI already set up. */ +export async function listOpencodeProviders(): Promise { + const res = await fetch('/api/ai/opencode/providers'); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body?.error || `OpenCode returned ${res.status}`); + return (body?.providers ?? []) as OpencodeProviderOption[]; +} + +/** The key is relayed to OpenCode's own credential store, never held by this + * app — same reasoning as the module note above, extended to provider setup. */ +export async function addOpencodeProvider(providerID: string, key: string): Promise { + const res = await fetch('/api/ai/opencode/providers', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ providerID, key }), + }); + const body = await res.json().catch(() => ({})); + // A 200 with `ok: false` means the route VERIFIED the write and the + // provider still isn't connected (some need more than one credential + // field — see the route's own comment) - that is as much a failure as a + // non-2xx status and must not be swallowed as success. + if (!res.ok || body?.ok === false) throw new Error(body?.error || `OpenCode returned ${res.status}`); +} + +export async function removeOpencodeProvider(providerID: string): Promise { + const res = await fetch(`/api/ai/opencode/providers?providerID=${encodeURIComponent(providerID)}`, { method: 'DELETE' }); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body?.error || `OpenCode returned ${res.status}`); +} + export async function chatOpencode(model: string, messages: ChatMessage[]): Promise { const res = await fetch('/api/ai/opencode/chat', { method: 'POST', diff --git a/lib/ai/opencode.ts b/lib/ai/opencode.ts index 1546f1b9..a3d62f40 100644 --- a/lib/ai/opencode.ts +++ b/lib/ai/opencode.ts @@ -131,6 +131,79 @@ export async function findOpencodeServer(): Promise<{ baseUrl: string; models: O return null; } +// ── Provider management ────────────────────────────────────────────────── +// +// Without this, "any LLM OpenCode supports" was only true for whatever the +// user had already authenticated via its own CLI (`opencode auth login`) — +// this app could pick a model, never add a provider. `GET /provider` lists +// every provider opencode KNOWS about (180 on a real run) with a `connected` +// array naming which ones actually have credentials; `GET /provider/auth` +// says which auth METHODS each one accepts. +// +// Scoped to API-key auth only for now, deliberately. `PUT /auth/{id}` with +// `{type:'api', key}` is one HTTP call with a schema-verified shape (tested +// live: 200, and the key round-trips into opencode's own auth.json). OAuth +// entries in `/provider/auth` (`{type:'oauth', label, prompts?}`) need a +// browser redirect + callback this app has no page for yet, and some carry +// interactive prompts (GitHub Copilot's deployment-type picker) beyond a +// single form — real scope for later, not something to half-build tonight. +// Providers offering only OAuth are still LISTED, just marked unsupported +// here, so the picker is honest about what it can and can't do. + +export interface OpencodeProviderInfo { + id: string; + name: string; + connected: boolean; + /** Whether this app can authenticate it — see the module note above. */ + supportsApiKey: boolean; +} + +interface ProviderListResponse { + all?: Array<{ id?: string; name?: string }>; + connected?: string[]; +} + +type ProviderAuthMethod = { type?: string }; +type ProviderAuthResponse = Record; + +export async function listOpencodeProviders(baseUrl: string): Promise { + const [providers, authMethods] = await Promise.all([ + fetchJson(`${baseUrl}/provider`, {}, PROBE_TIMEOUT_MS) as Promise, + fetchJson(`${baseUrl}/provider/auth`, {}, PROBE_TIMEOUT_MS) as Promise, + ]); + if (!providers || !Array.isArray(providers.all)) return []; + const connected = new Set(providers.connected ?? []); + return providers.all + .filter((p): p is { id: string; name?: string } => typeof p?.id === 'string' && !!p.id) + .map((p) => ({ + id: p.id, + name: p.name || p.id, + connected: connected.has(p.id), + supportsApiKey: (authMethods?.[p.id] ?? []).some((m) => m.type === 'api'), + })) + .sort((a, b) => (a.connected === b.connected ? a.name.localeCompare(b.name) : a.connected ? -1 : 1)); +} + +/** Stores an API key for a provider. Throws with opencode's own status on + * failure rather than returning a boolean, so the route can pass a real + * error back instead of a bare "didn't work". */ +export async function setOpencodeProviderKey(baseUrl: string, providerID: string, key: string): Promise { + const res = await fetch(`${baseUrl}/auth/${encodeURIComponent(providerID)}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json', ...authHeaders() }, + body: JSON.stringify({ type: 'api', key }), + }); + if (!res.ok) throw new Error(`OpenCode rejected the credential (HTTP ${res.status})`); +} + +export async function removeOpencodeProvider(baseUrl: string, providerID: string): Promise { + const res = await fetch(`${baseUrl}/auth/${encodeURIComponent(providerID)}`, { + method: 'DELETE', + headers: authHeaders(), + }); + if (!res.ok) throw new Error(`OpenCode could not remove the credential (HTTP ${res.status})`); +} + interface OpencodeMessageResponse { parts?: Array<{ type?: string; text?: string }>; }