SRCmail/e2e/electron-ai-local-index.spec.ts

import { test, expect, _electron as electron } from '@playwright/test';
import type { ElectronApplication, Page } from '@playwright/test';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';

/**
 * Proves the two hardest-to-fake claims about the AI Assistant's `local`
 * class in the REAL packaged desktop shell, not a browser tab:
 *
 *   1. The LLM genuinely runs locally — a direct browser-side fetch to
 *      this machine's own Ollama (127.0.0.1:11434), never proxied through
 *      this app's backend.
 *   2. It is genuinely grounded in the ENCRYPTED LOCAL SQLITE/FTS5 MAIL
 *      INDEX (lib/mail-index/**), not the separate real-JMAP-embeddings
 *      server leg (lib/ai/retrieval/mail-embeddings.ts) — AI_SERVER_BASE_URL
 *      is deliberately left UNSET here so only the local FTS leg can
 *      supply retrieval context. If this test passes, the local index
 *      leg is the only possible source of the grounded answer.
 *
 * Needs a real launch through electron/main.ts's startStandaloneServer(),
 * not ELECTRON_LOAD_URL — that's the only code path that wires up the
 * fd-3 key channel / safeStorage the encrypted index depends on (see
 * integration/tests/12-electron-mail-index.spec.ts's header for the full
 * reasoning). That function picks a random free port every launch, which
 * would make it impossible to also point DEV_MOCK_JMAP's JMAP_SERVER_URL
 * at this same server's own /api/dev-jmap route — hence
 * VNCMAIL_TEST_FIXED_PORT, a narrow, off-by-default escape hatch added to
 * electron/main.ts specifically to make this test possible without a real
 * Stalwart fixture.
 *
 * Requires a real Ollama already running on this machine with at least one
 * completion-capable model installed — skips (not fails) otherwise, since
 * "no local LLM on this machine" is an environment fact, not a bug.
 */

const projectRoot = path.resolve(__dirname, '..');
const FIXED_PORT = 39217;
const ORIGIN = `http://127.0.0.1:${FIXED_PORT}`;

async function ollamaIsUp(): Promise<boolean> {
  try {
    const res = await fetch('http://127.0.0.1:11434/api/tags');
    if (!res.ok) return false;
    const body = (await res.json()) as { models?: Array<{ capabilities?: string[] }> };
    return (body.models ?? []).some((m) => !m.capabilities || m.capabilities.includes('completion'));
  } catch {
    return false;
  }
}

test.describe('Electron desktop shell - local LLM answers from the real encrypted mail index', () => {
  let electronApp: ElectronApplication;
  let appWindow: Page;
  let userDataDir: string;

  test.beforeAll(async () => {
    if (!(await ollamaIsUp())) {
      test.skip(true, 'No local Ollama with a completion-capable model reachable on this machine — environment fact, not a failure.');
    }

    userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-ai-index-test-'));

    electronApp = await electron.launch({
      args: [projectRoot, `--user-data-dir=${userDataDir}`],
      env: {
        ...process.env,
        VNCMAIL_TEST_FIXED_PORT: String(FIXED_PORT),
        DEV_MOCK_JMAP: 'true',
        JMAP_SERVER_URL: `${ORIGIN}/api/dev-jmap`,
        // Deliberately NO SESSION_SECRET: the desktop shell must supply its
        // own per-install secret (electron/main.ts's ensureSessionSecretFile)
        // or the auth-context cookie can never be minted and every index
        // route 401s. Up to 1.7.8 the packaged app shipped exactly that way,
        // and every test masked it by injecting a secret here — this test
        // now proves the shell stands on its own.
        SESSION_SECRET: '',
        // Deliberately UNSET: isolates grounding to the local FTS leg (see
        // module header) — the server embeddings leg 404s cleanly instead
        // of silently also being able to answer the question.
        AI_SERVER_BASE_URL: '',
        NODE_ENV: 'production',
      },
    });

    appWindow = await electronApp.firstWindow();
    await appWindow.waitForLoadState('domcontentloaded');
  });

  test.afterAll(async () => {
    await electronApp?.close();
    if (userDataDir) fs.rmSync(userDataDir, { recursive: true, force: true });
  });

  test('logs in, builds the real encrypted index, and a local Ollama model answers a mail question grounded in it', async () => {
    // ── 1. Real dev-mode login (sets the real session cookie the offline
    // index and every other server-side-identity feature need). ──
    const devLoginContainer = appWindow.locator('div', { hasText: 'Dev mode - logging in as dev@localhost' }).last();
    await devLoginContainer.getByRole('button').click();
    await appWindow.waitForURL((url) => !url.pathname.includes('login'), { timeout: 20000 });

    // ── 2. Wait for the AUTOMATIC boot catch-up to build the encrypted
    // index — NO manual /api/offline/sync or /api/offline/reindex calls.
    // This is the load-bearing change from this spec's first version: the
    // real user experience is "log in, index appears on its own", and the
    // first version's manual calls proved only that the plumbing COULD
    // work, not that anything actually drives it. page.tsx schedules the
    // first attempt ~4s after the authenticated mail page mounts (with
    // retries at 20s/60s for the login race), so poll generously. ──
    await expect
      .poll(
        async () =>
          appWindow.evaluate(async () => {
            const res = await fetch(`/api/offline/search?q=${encodeURIComponent('Villa sul Lago check-in')}&limit=6`);
            if (!res.ok) return `http-${res.status}`;
            const body = await res.json().catch(() => null);
            const hits = (body?.hits ?? []) as Array<{ title?: string }>;
            return hits.some((h) => /villa sul lago/i.test(h.title ?? '')) ? 'hit' : 'indexed-but-empty';
          }),
        {
          timeout: 90000,
          intervals: [2000],
          message:
            'the boot catch-up (page.tsx) must build the index automatically after login — http-401 here means the auth-context cookie was never minted (the missing-SESSION_SECRET class of bug), http-404 means the key channel/store dir never activated',
        },
      )
      .toBe('hit');

    // ── 3. The new toolbar entry point must be present in the main mail
    // view — the AI Assistant is a feature of the app, not of the Settings
    // page. ──
    await expect(appWindow.getByRole('button', { name: 'AI Assistant' })).toBeVisible({ timeout: 10000 });

    // ── 4. Navigate to the real AI Assistant settings UI and use the
    // local-discovery "Connect" banner — the exact flow a real user takes,
    // proving discovery -> connect -> ask works as one integrated feature.
    // Deliberately an in-app SPA navigation (click the real sidebar link),
    // NOT appWindow.goto() — a full page reload drops whatever client-only
    // session state the dev-mode login established (confirmed: goto('/settings')
    // bounces straight back to /login even though the JMAP session cookie
    // from step 2/3 is still valid), so the click is load-bearing, not
    // cosmetic. ──
    await appWindow.locator('a[href="/settings"], a[href*="/settings"]').first().click();
    const searchBox = appWindow.locator('input[type="search"]').first();
    await searchBox.fill('AI Assistant');
    await appWindow.getByRole('button', { name: 'AI Assistant' }).click();

    const connectButton = appWindow.getByRole('button', { name: /Connect/i });
    await expect(connectButton, 'the local-discovery banner should appear since a real Ollama is running on this machine').toBeVisible({ timeout: 10000 });
    const bannerText = await appWindow.locator('text=Local AI found on this machine').locator('..').innerText();
    console.log('[ai-local-index] discovery banner text:', bannerText);
    await connectButton.click();

    // ── 5. Ask a question only answerable by combining the local LLM
    // with the local index's actual content. ──
    const questionBox = appWindow.getByPlaceholder(/What did legal say/i);
    await questionBox.fill('When is check-in for the Villa sul Lago booking, and what time?');
    const askButton = appWindow.getByRole('button', { name: /^Ask$/ });
    await expect(askButton, 'Ask must be enabled immediately after Connect pre-fills provider+model').toBeEnabled({ timeout: 5000 });

    const chatRequests: string[] = [];
    const offlineSearchCalls: Array<{ url: string; status: number; body: unknown }> = [];
    appWindow.on('request', (req) => {
      if (req.url().includes('11434')) chatRequests.push(`${req.method()} ${req.url()}`);
    });
    appWindow.on('response', async (res) => {
      if (res.url().includes('/api/offline/search')) {
        offlineSearchCalls.push({ url: res.url(), status: res.status(), body: await res.json().catch(() => null) });
      }
    });

    // Log the exact prompt Ollama actually received, straight from the
    // request body — the ground truth for "did retrieval even fire".
    const ollamaChatPayloads: unknown[] = [];
    await appWindow.route('**/api/chat', async (route) => {
      try {
        ollamaChatPayloads.push(JSON.parse(route.request().postData() ?? 'null'));
      } catch { /* ignore parse failure, still let the request through */ }
      await route.continue();
    });

    await askButton.click();

    const answerLocator = appWindow.locator('p.whitespace-pre-wrap').first();
    await expect(answerLocator, 'the local Ollama model should produce an answer within a generous timeout').toBeVisible({ timeout: 60000 });
    const answerText = await answerLocator.innerText();
    console.log('[ai-local-index] final answer:', answerText);
    console.log('[ai-local-index] direct-to-Ollama requests observed:', chatRequests);
    console.log('[ai-local-index] /api/offline/search calls during Ask:', JSON.stringify(offlineSearchCalls));
    console.log('[ai-local-index] exact payload(s) sent to Ollama /api/chat:', JSON.stringify(ollamaChatPayloads));

    // The real proof: the model's own words contain the fact that only
    // exists in the indexed email (28 March, 15:00), and the request log
    // shows the renderer talked to Ollama's loopback address directly.
    expect(answerText).toMatch(/28\s*march|march\s*28/i);
    expect(answerText).toMatch(/15:00|3\s*pm|3:00\s*pm/i);
    expect(chatRequests.some((r) => r.includes('/api/chat')), `expected a direct renderer -> Ollama /api/chat request, saw: ${JSON.stringify(chatRequests)}`).toBe(true);

    await appWindow.screenshot({ path: path.join(projectRoot, 'electron-ai-local-index-result.png'), fullPage: true });
  });
});