Found by a new real Electron e2e test built specifically to prove the `local` AI class genuinely works end-to-end in the packaged desktop shell: a real local Ollama model answering a real question, grounded in the real encrypted SQLite/FTS5 mail index — not a browser tab, not a mock. First run surfaced a genuine bug: toFtsMatchQuery() AND-joins every token, which is right for a deliberate search-box query but wrong for the natural- language questions the AI retrieval surface (/api/offline/search — see its own module header, "THE RETRIEVAL SURFACE") actually receives. "When is check-in for the Villa sul Lago booking, and what time?" shares almost none of its own function words with the email that answers it, so ANDing every token — including "when"/"is"/"for"/"the"/"and"/"what" — returned 0 hits against an index that correctly returns the right email for "Villa sul Lago check-in". Fix: new toFtsMatchQueryAny() (lib/mail-index/store.ts) — drops a small, well-known English stop-word list, OR-joins what's left, and lets the existing bm25 ranking pick the winner among partial matches. Deliberately a NEW function, not a change to toFtsMatchQuery itself: that one's own tests rely on "AND"/"OR"/"NOT" surviving verbatim as literal search terms (FTS5-keyword-injection safety) — a different guarantee than this one's job of turning a question into a good search. search() gains a `mode: 'and' | 'any'` option (default 'and', so every existing caller is unaffected); the offline-search route passes 'any', since its one real caller is exactly this AI-question shape. Also added, to make the e2e test possible at all: electron/main.ts's VNCMAIL_TEST_FIXED_PORT — a narrow, off-by-default escape hatch so DEV_MOCK_JMAP's JMAP_SERVER_URL can point at this same standalone server's own /api/dev-jmap. Needed because the encrypted index's key channel (fd-3/safeStorage) only gets wired up in startStandaloneServer()'s own random-port launch path, never when ELECTRON_LOAD_URL bypasses it for a plain `next dev` target — so this was the only way to exercise the real index without a full Stalwart+SMTP Docker fixture. Verified live in the real packaged Electron shell, not just unit tests: real dev-mode login, real multi-round /api/offline/sync + /api/offline/reindex (39 mail/35 calendar/23 contacts indexed), real local-discovery banner (11 real Ollama models on this machine), real "Connect", a real question through the real Settings UI, a real direct renderer->Ollama /api/chat call (confirmed via network log, never proxied through this app's backend), and the model's own answer citing the exact right fact: "Saturday 28 March at 15:00" — a fact that exists nowhere except in the one indexed email. 4 new unit tests for toFtsMatchQueryAny. Full gate: tsc clean, eslint clean, 2502/2502 tests passing, build clean, e2e/electron-ai-local-index.spec.ts passing against the real standalone server + real Electron + real Ollama.
123 lines
4.9 KiB
TypeScript
123 lines
4.9 KiB
TypeScript
// GET /api/offline/search?q=...&types=mail,calendar&limit=20
|
|
//
|
|
// THE RETRIEVAL SURFACE. This is what an AI/RAG feature calls to gather
|
|
// relevant context from the user's own mail, calendar, contacts and files
|
|
// before prompting a model - hence the `snippet` on every hit and the
|
|
// `contextBlock` convenience field, which is the same information already
|
|
// flattened into text a prompt can carry directly.
|
|
//
|
|
// Read-only: it never touches the network and never writes. Gated identically
|
|
// to the reindex route.
|
|
import { NextRequest, NextResponse } from 'next/server';
|
|
import { logger } from '@/lib/logger';
|
|
import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
|
import { hasKeyChannel, IndexKeyError, withIndexKey } from '@/lib/mail-index/key';
|
|
import { getStoreDir } from '@/lib/mail-index/paths';
|
|
import { IndexSessionError, resolveIndexSession } from '@/lib/mail-index/reindex';
|
|
import {
|
|
isContentType, MailIndex, MailIndexUnavailableError, type ContentType, type SearchHit,
|
|
} from '@/lib/mail-index/store';
|
|
|
|
export const runtime = 'nodejs';
|
|
export const dynamic = 'force-dynamic';
|
|
|
|
/**
|
|
* One hit as a plain text block, ready to be concatenated into a prompt.
|
|
* Kept server-side so every caller (a chat feature, a future agent, a test)
|
|
* formats context the same way rather than each inventing its own.
|
|
*/
|
|
function toContextBlock(hit: SearchHit): string {
|
|
const label: Record<ContentType, string> = {
|
|
mail: 'EMAIL', calendar: 'CALENDAR EVENT', contact: 'CONTACT', file: 'FILE',
|
|
};
|
|
const lines = [`[${label[hit.contentType]}] ${hit.title}`];
|
|
if (hit.occurredAt) lines.push(`Date: ${hit.occurredAt}`);
|
|
if (hit.people) lines.push(`People: ${hit.people}`);
|
|
const path = hit.metadata?.path;
|
|
if (typeof path === 'string' && path) lines.push(`Path: ${path}`);
|
|
if (hit.snippet) lines.push(`Excerpt: ${hit.snippet}`);
|
|
return lines.join('\n');
|
|
}
|
|
|
|
export async function GET(request: NextRequest) {
|
|
if (!getStoreDir()) {
|
|
return new NextResponse(null, { status: 404 });
|
|
}
|
|
if (!hasKeyChannel() || !isSqlcipherAvailable()) {
|
|
return NextResponse.json(
|
|
{ error: 'Encrypted local index is unavailable in this process.', code: 'unavailable' },
|
|
{ status: 503 },
|
|
);
|
|
}
|
|
|
|
const params = request.nextUrl.searchParams;
|
|
const query = (params.get('q') ?? '').trim();
|
|
const wantStats = params.get('stats') === 'true';
|
|
|
|
if (!query && !wantStats) {
|
|
return NextResponse.json({ error: 'Missing q parameter' }, { status: 400 });
|
|
}
|
|
if (query.length > 512) {
|
|
return NextResponse.json({ error: 'Query too long' }, { status: 400 });
|
|
}
|
|
|
|
const types = (params.get('types') ?? '')
|
|
.split(',')
|
|
.map((t) => t.trim())
|
|
.filter(isContentType);
|
|
|
|
const limitRaw = Number(params.get('limit') ?? '20');
|
|
const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(Math.trunc(limitRaw), 1), 100) : 20;
|
|
|
|
try {
|
|
const session = await resolveIndexSession(request);
|
|
const storeDir = getStoreDir();
|
|
if (!storeDir) return new NextResponse(null, { status: 404 });
|
|
|
|
const payload = await withIndexKey(session.accountId, (key) => {
|
|
const index = MailIndex.open({ storeDir, accountId: session.accountId, key });
|
|
try {
|
|
const stats = index.stats();
|
|
if (!query) return { hits: [] as SearchHit[], stats };
|
|
// 'any': this route is the AI/RAG retrieval surface (see module
|
|
// header) - its one real caller sends natural-language questions,
|
|
// not deliberate search-box keywords, so strict AND-every-token
|
|
// matching (the default) drops nearly all of them. See
|
|
// toFtsMatchQueryAny's docstring for the confirmed-live failure.
|
|
return { hits: index.search({ query, types, limit, mode: 'any' }), stats: wantStats ? stats : undefined };
|
|
} finally {
|
|
index.close();
|
|
}
|
|
});
|
|
|
|
return NextResponse.json(
|
|
{
|
|
ok: true,
|
|
query,
|
|
types: types.length > 0 ? types : 'all',
|
|
count: payload.hits.length,
|
|
hits: payload.hits,
|
|
// Everything a prompt needs, pre-joined in rank order.
|
|
contextBlock: payload.hits.map(toContextBlock).join('\n\n---\n\n'),
|
|
...(payload.stats ? { stats: payload.stats } : {}),
|
|
},
|
|
{ headers: { 'Cache-Control': 'no-store' } },
|
|
);
|
|
} catch (error) {
|
|
if (error instanceof IndexSessionError) {
|
|
return NextResponse.json({ error: error.message }, { status: error.status });
|
|
}
|
|
if (error instanceof IndexKeyError) {
|
|
const status = error.code === 'no-secure-storage' ? 503 : 500;
|
|
return NextResponse.json({ error: error.message, code: error.code }, { status });
|
|
}
|
|
if (error instanceof MailIndexUnavailableError) {
|
|
return NextResponse.json({ error: error.message, code: 'unavailable' }, { status: 503 });
|
|
}
|
|
logger.error('mail-index search failed', {
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
return NextResponse.json({ error: 'Search failed' }, { status: 500 });
|
|
}
|
|
}
|