import { test, expect, _electron as electron } from '@playwright/test'; import type { ElectronApplication, Page } from '@playwright/test'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; /** * Proves the two hardest-to-fake claims about the AI Assistant's `local` * class in the REAL packaged desktop shell, not a browser tab: * * 1. The LLM genuinely runs locally — a direct browser-side fetch to * this machine's own Ollama (127.0.0.1:11434), never proxied through * this app's backend. * 2. It is genuinely grounded in the ENCRYPTED LOCAL SQLITE/FTS5 MAIL * INDEX (lib/mail-index/**), not the separate real-JMAP-embeddings * server leg (lib/ai/retrieval/mail-embeddings.ts) — AI_SERVER_BASE_URL * is deliberately left UNSET here so only the local FTS leg can * supply retrieval context. If this test passes, the local index * leg is the only possible source of the grounded answer. * * Needs a real launch through electron/main.ts's startStandaloneServer(), * not ELECTRON_LOAD_URL — that's the only code path that wires up the * fd-3 key channel / safeStorage the encrypted index depends on (see * integration/tests/12-electron-mail-index.spec.ts's header for the full * reasoning). That function picks a random free port every launch, which * would make it impossible to also point DEV_MOCK_JMAP's JMAP_SERVER_URL * at this same server's own /api/dev-jmap route — hence * VNCMAIL_TEST_FIXED_PORT, a narrow, off-by-default escape hatch added to * electron/main.ts specifically to make this test possible without a real * Stalwart fixture. * * Requires a real Ollama already running on this machine with at least one * completion-capable model installed — skips (not fails) otherwise, since * "no local LLM on this machine" is an environment fact, not a bug. */ const projectRoot = path.resolve(__dirname, '..'); const FIXED_PORT = 39217; const ORIGIN = `http://127.0.0.1:${FIXED_PORT}`; async function ollamaIsUp(): Promise { try { const res = await fetch('http://127.0.0.1:11434/api/tags'); if (!res.ok) return false; const body = (await res.json()) as { models?: Array<{ capabilities?: string[] }> }; return (body.models ?? []).some((m) => !m.capabilities || m.capabilities.includes('completion')); } catch { return false; } } test.describe('Electron desktop shell - local LLM answers from the real encrypted mail index', () => { let electronApp: ElectronApplication; let appWindow: Page; let userDataDir: string; test.beforeAll(async () => { if (!(await ollamaIsUp())) { test.skip(true, 'No local Ollama with a completion-capable model reachable on this machine — environment fact, not a failure.'); } userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-ai-index-test-')); electronApp = await electron.launch({ args: [projectRoot, `--user-data-dir=${userDataDir}`], env: { ...process.env, VNCMAIL_TEST_FIXED_PORT: String(FIXED_PORT), DEV_MOCK_JMAP: 'true', JMAP_SERVER_URL: `${ORIGIN}/api/dev-jmap`, // Deliberately NO SESSION_SECRET: the desktop shell must supply its // own per-install secret (electron/main.ts's ensureSessionSecretFile) // or the auth-context cookie can never be minted and every index // route 401s. Up to 1.7.8 the packaged app shipped exactly that way, // and every test masked it by injecting a secret here — this test // now proves the shell stands on its own. SESSION_SECRET: '', // Deliberately UNSET: isolates grounding to the local FTS leg (see // module header) — the server embeddings leg 404s cleanly instead // of silently also being able to answer the question. AI_SERVER_BASE_URL: '', NODE_ENV: 'production', }, }); appWindow = await electronApp.firstWindow(); await appWindow.waitForLoadState('domcontentloaded'); }); test.afterAll(async () => { await electronApp?.close(); if (userDataDir) fs.rmSync(userDataDir, { recursive: true, force: true }); }); test('logs in, builds the real encrypted index, and a local Ollama model answers a mail question grounded in it', async () => { // ── 1. Real dev-mode login (sets the real session cookie the offline // index and every other server-side-identity feature need). ── const devLoginContainer = appWindow.locator('div', { hasText: 'Dev mode - logging in as dev@localhost' }).last(); await devLoginContainer.getByRole('button').click(); await appWindow.waitForURL((url) => !url.pathname.includes('login'), { timeout: 20000 }); // ── 2. Wait for the AUTOMATIC boot catch-up to build the encrypted // index — NO manual /api/offline/sync or /api/offline/reindex calls. // This is the load-bearing change from this spec's first version: the // real user experience is "log in, index appears on its own", and the // first version's manual calls proved only that the plumbing COULD // work, not that anything actually drives it. page.tsx schedules the // first attempt ~4s after the authenticated mail page mounts (with // retries at 20s/60s for the login race), so poll generously. ── await expect .poll( async () => appWindow.evaluate(async () => { const res = await fetch(`/api/offline/search?q=${encodeURIComponent('Villa sul Lago check-in')}&limit=6`); if (!res.ok) return `http-${res.status}`; const body = await res.json().catch(() => null); const hits = (body?.hits ?? []) as Array<{ title?: string }>; return hits.some((h) => /villa sul lago/i.test(h.title ?? '')) ? 'hit' : 'indexed-but-empty'; }), { timeout: 90000, intervals: [2000], message: 'the boot catch-up (page.tsx) must build the index automatically after login — http-401 here means the auth-context cookie was never minted (the missing-SESSION_SECRET class of bug), http-404 means the key channel/store dir never activated', }, ) .toBe('hit'); // ── 3. The new toolbar entry point must be present in the main mail // view — the AI Assistant is a feature of the app, not of the Settings // page. ── await expect(appWindow.getByRole('button', { name: 'AI Assistant' })).toBeVisible({ timeout: 10000 }); // ── 4. Navigate to the real AI Assistant settings UI and use the // local-discovery "Connect" banner — the exact flow a real user takes, // proving discovery -> connect -> ask works as one integrated feature. // Deliberately an in-app SPA navigation (click the real sidebar link), // NOT appWindow.goto() — a full page reload drops whatever client-only // session state the dev-mode login established (confirmed: goto('/settings') // bounces straight back to /login even though the JMAP session cookie // from step 2/3 is still valid), so the click is load-bearing, not // cosmetic. ── await appWindow.locator('a[href="/settings"], a[href*="/settings"]').first().click(); const searchBox = appWindow.locator('input[type="search"]').first(); await searchBox.fill('AI Assistant'); await appWindow.getByRole('button', { name: 'AI Assistant' }).click(); const connectButton = appWindow.getByRole('button', { name: /Connect/i }); await expect(connectButton, 'the local-discovery banner should appear since a real Ollama is running on this machine').toBeVisible({ timeout: 10000 }); const bannerText = await appWindow.locator('text=Local AI found on this machine').locator('..').innerText(); console.log('[ai-local-index] discovery banner text:', bannerText); await connectButton.click(); // ── 5. Ask a question only answerable by combining the local LLM // with the local index's actual content. ── const questionBox = appWindow.getByPlaceholder(/What did legal say/i); await questionBox.fill('When is check-in for the Villa sul Lago booking, and what time?'); const askButton = appWindow.getByRole('button', { name: /^Ask$/ }); await expect(askButton, 'Ask must be enabled immediately after Connect pre-fills provider+model').toBeEnabled({ timeout: 5000 }); const chatRequests: string[] = []; const offlineSearchCalls: Array<{ url: string; status: number; body: unknown }> = []; appWindow.on('request', (req) => { if (req.url().includes('11434')) chatRequests.push(`${req.method()} ${req.url()}`); }); appWindow.on('response', async (res) => { if (res.url().includes('/api/offline/search')) { offlineSearchCalls.push({ url: res.url(), status: res.status(), body: await res.json().catch(() => null) }); } }); // Log the exact prompt Ollama actually received, straight from the // request body — the ground truth for "did retrieval even fire". const ollamaChatPayloads: unknown[] = []; await appWindow.route('**/api/chat', async (route) => { try { ollamaChatPayloads.push(JSON.parse(route.request().postData() ?? 'null')); } catch { /* ignore parse failure, still let the request through */ } await route.continue(); }); await askButton.click(); const answerLocator = appWindow.locator('p.whitespace-pre-wrap').first(); await expect(answerLocator, 'the local Ollama model should produce an answer within a generous timeout').toBeVisible({ timeout: 60000 }); const answerText = await answerLocator.innerText(); console.log('[ai-local-index] final answer:', answerText); console.log('[ai-local-index] direct-to-Ollama requests observed:', chatRequests); console.log('[ai-local-index] /api/offline/search calls during Ask:', JSON.stringify(offlineSearchCalls)); console.log('[ai-local-index] exact payload(s) sent to Ollama /api/chat:', JSON.stringify(ollamaChatPayloads)); // The real proof: the model's own words contain the fact that only // exists in the indexed email (28 March, 15:00), and the request log // shows the renderer talked to Ollama's loopback address directly. expect(answerText).toMatch(/28\s*march|march\s*28/i); expect(answerText).toMatch(/15:00|3\s*pm|3:00\s*pm/i); expect(chatRequests.some((r) => r.includes('/api/chat')), `expected a direct renderer -> Ollama /api/chat request, saw: ${JSON.stringify(chatRequests)}`).toBe(true); await appWindow.screenshot({ path: path.join(projectRoot, 'electron-ai-local-index-result.png'), fullPage: true }); }); });