import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { randomBytes } from 'node:crypto'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { isSqlcipherAvailable } from '../binding'; import type { SqlcipherConstructor, SqlcipherDatabase, SqlcipherStatement } from '../binding'; import { accountFileToken, getStoreDir, indexDbPath, STORE_DIR_ENV } from '../paths'; import { MailIndex, openKeyed, toFtsMatchQuery, toFtsMatchQueryAny, type IndexDoc } from '../store'; describe('toFtsMatchQuery', () => { it('quotes every token so FTS5 operators in user input cannot break the query', () => { // FTS5's MATCH grammar is NOT protected by SQL parameter binding: a bare // quote or a stray NEAR/AND/* raises `fts5: syntax error`, which would turn // a search box into a 500. expect(toFtsMatchQuery('a" OR b')).toBe('"a" AND "OR" AND "b"'); // No trailing `*` here: the final token is one character, below the // prefix-match threshold (see the next test). expect(toFtsMatchQuery('NEAR(x y)')).toBe('"NEAR" AND "x" AND "y"'); expect(toFtsMatchQuery('NEAR(x yes)')).toBe('"NEAR" AND "x" AND "yes"*'); expect(toFtsMatchQuery('foo*')).toBe('"foo"*'); expect(toFtsMatchQuery('a AND NOT b')).toContain('"NOT"'); }); it('prefix-matches only the final token, and only when it is long enough', () => { expect(toFtsMatchQuery('zurich lea')).toBe('"zurich" AND "lea"*'); // Two characters would match too much of a mailbox to be useful. expect(toFtsMatchQuery('zurich le')).toBe('"zurich" AND "le"'); }); it('keeps unicode letters, emails and hyphenated words', () => { expect(toFtsMatchQuery('Müller')).toBe('"Müller"*'); expect(toFtsMatchQuery('東京')).toBe('"東京"'); expect(toFtsMatchQuery('a@b.com')).toBe('"a@b.com"*'); expect(toFtsMatchQuery("O'Brien-Smith")).toBe('"O\'Brien-Smith"*'); }); it('returns null for input with no usable tokens', () => { expect(toFtsMatchQuery('')).toBeNull(); expect(toFtsMatchQuery(' ')).toBeNull(); expect(toFtsMatchQuery('***')).toBeNull(); expect(toFtsMatchQuery(undefined as unknown as string)).toBeNull(); }); it('bounds the token count', () => { const many = Array.from({ length: 100 }, (_, i) => `w${i}`).join(' '); expect((toFtsMatchQuery(many) ?? '').split(' AND ')).toHaveLength(24); }); }); describe('toFtsMatchQueryAny', () => { it('drops English function words and OR-joins what is left - the confirmed-live failure this fixes', () => { // AND-every-token (toFtsMatchQuery) returns 0 hits for this exact // question against a document that only contains "Villa sul Lago" and // "check-in" - see app/api/offline/search/route.ts's comment and the // e2e electron-ai-local-index.spec.ts run that first caught this. const result = toFtsMatchQueryAny('When is check-in for the Villa sul Lago booking, and what time?'); expect(result).not.toBeNull(); expect(result).not.toContain(' AND '); expect(result).toContain('"check-in"'); expect(result).toContain('"Villa"'); expect(result).toContain('"sul"'); expect(result).toContain('"Lago"'); expect(result).toContain('"booking"'); // "time" is the last surviving content word, so it gets the // prefix-match star - not "Lago", which is merely the last one this // test happens to name first. expect(result).toContain('"time"*'); // Pure stop words, correctly dropped rather than OR-joined as noise that // would otherwise match almost every document in a mailbox. expect(result).not.toMatch(/"When"|"is"|"for"|"the"|"and"|"what"/i); }); it('falls back to the unfiltered text when every word is a stop word, rather than searching for nothing', () => { // "What is this" is 100% stop words - dropping all of them would leave // zero tokens (a null match, meaning "return everything" is wrong for a // question shaped like this); falling back to the original text at // least keeps a real, if weak, query. const result = toFtsMatchQueryAny('What is this'); expect(result).not.toBeNull(); }); it('still safely quotes FTS5 syntax characters even after stop-word filtering removes the surrounding noise', () => { // "OR"/"NEAR" themselves are common enough as English words that this // builder's stop-word list intentionally drops bare "or" (unlike // toFtsMatchQuery, which preserves it verbatim - see that test's own // comment on why: different concern, different guarantee). The safety // property that DOES still apply here is the one that matters for a // 500: whatever tokens survive filtering are always quoted before // reaching FTS5, so a stray `"`/`*`/`(` in real question text can never // raise a syntax error. const result = toFtsMatchQueryAny('a" NEAR(bar) baz*'); expect(result).not.toBeNull(); expect(result).toContain('"NEAR"'); expect(result).toContain('"bar"'); expect(result).toContain('"baz"'); expect(result).not.toMatch(/fts5|syntax/i); }); it('returns null for input with no usable tokens', () => { expect(toFtsMatchQueryAny('')).toBeNull(); expect(toFtsMatchQueryAny('***')).toBeNull(); expect(toFtsMatchQueryAny(undefined as unknown as string)).toBeNull(); }); }); describe('paths', () => { const original = process.env[STORE_DIR_ENV]; afterEach(() => { if (original === undefined) delete process.env[STORE_DIR_ENV]; else process.env[STORE_DIR_ENV] = original; }); it('is disabled unless the env var is set - the hosted-deployment gate', () => { delete process.env[STORE_DIR_ENV]; expect(getStoreDir()).toBeNull(); process.env[STORE_DIR_ENV] = ''; expect(getStoreDir()).toBeNull(); }); it('rejects a relative path, which would resolve against the server cwd', () => { process.env[STORE_DIR_ENV] = 'offline'; expect(getStoreDir()).toBeNull(); process.env[STORE_DIR_ENV] = '/abs/offline'; expect(getStoreDir()).toBe('/abs/offline'); }); it('hashes the filename so the directory is not an account inventory', () => { const token = accountFileToken('linus@example.com'); expect(token).toMatch(/^[0-9a-f]{32}$/); expect(token).not.toContain('linus'); expect(indexDbPath('/s', 'linus@example.com')).toBe(`/s/index/${token}.db`); // Deterministic - the same account must resolve to the same file forever. expect(accountFileToken('linus@example.com')).toBe(token); }); }); /** A fake `SqlcipherDatabase` whose `pragma()` is driven by the given handler. */ function fakeSqlcipherCtor( pragmaHandler: (source: string) => unknown, ): { ctor: SqlcipherConstructor; instances: Array<{ closeCalls: number }> } { const instances: Array<{ closeCalls: number }> = []; function FakeDatabase(this: unknown, _path?: string): SqlcipherDatabase { const state = { closeCalls: 0 }; instances.push(state); const stmt: SqlcipherStatement = { run: () => ({ changes: 0, lastInsertRowid: 0 }), get: () => undefined, all: () => [] }; const db: SqlcipherDatabase = { exec: () => {}, prepare: () => stmt, pragma: pragmaHandler, close: () => { state.closeCalls += 1; }, }; return db; } return { ctor: FakeDatabase as unknown as SqlcipherConstructor, instances }; } describe('openKeyed', () => { it('closes the connection before rethrowing when a pragma AFTER assertEncrypted fails', () => { // The bug this guards: `journal_mode = WAL` (or any pragma after the key // check) throwing must not leak the native handle - a `try` that only // wrapped SOME of these calls previously let exactly this escape. const { ctor, instances } = fakeSqlcipherCtor((source) => { if (source === 'cipher_version') return [{ cipher_version: 'fake-4.5.0' }]; if (source === 'journal_mode = WAL') throw new Error('simulated pragma failure'); return undefined; }); expect(() => openKeyed(ctor, '/fake/path.db', randomBytes(32))).toThrow(/simulated pragma failure/); expect(instances).toHaveLength(1); expect(instances[0].closeCalls).toBe(1); }); it('tolerates the extra close when assertEncrypted itself is the failure (its own close is idempotent)', () => { // assertEncrypted closes on its own failure before throwing; openKeyed's // catch then calls close again. That second call must be harmless, not a // new crash - hence >= 1 rather than a fixed count. const { ctor, instances } = fakeSqlcipherCtor((source) => (source === 'cipher_version' ? [] : undefined)); expect(() => openKeyed(ctor, '/fake/path.db', randomBytes(32))).toThrow(/no SQLCipher support/); expect(instances[0].closeCalls).toBeGreaterThanOrEqual(1); }); }); function doc(overrides: Partial = {}): IndexDoc { return { jmapAccountId: 'acc1', contentType: 'mail', id: 'M1', title: 'Quarterly budget review', people: 'Sophie Müller sophie@example.com', body: 'The Zurich office lease renewal needs a decision before September.', occurredAt: '2026-08-01T10:00:00Z', metadata: { threadId: 'T1' }, ...overrides, }; } // The native binding is an OPTIONAL dependency, so these skip rather than fail // on a platform with no prebuild (e.g. Alpine/musl in CI containers). describe.skipIf(!isSqlcipherAvailable())('MailIndex (real SQLCipher)', () => { let storeDir: string; const accountId = 'linus@example.com'; const key = randomBytes(32); beforeEach(() => { storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mail-index-test-')); }); afterEach(() => { fs.rmSync(storeDir, { recursive: true, force: true }); }); const open = () => MailIndex.open({ storeDir, accountId, key }); it('writes an ENCRYPTED file - no plaintext recoverable from the raw bytes', () => { const index = open(); index.upsert([doc()]); index.close(); const bytes = fs.readFileSync(indexDbPath(storeDir, accountId)); // The canary check, not just a header check: this is the assertion that // would have caught `PRAGMA key` being a silent no-op. expect(bytes.includes('Zurich office lease')).toBe(false); expect(bytes.includes('Quarterly budget')).toBe(false); expect(bytes.subarray(0, 15).toString('latin1')).not.toBe('SQLite format 3'); }); it('rejects a wrong key and rebuilds instead of throwing at the caller', () => { const index = open(); index.upsert([doc()]); index.close(); // A different key cannot read the data; the store recreates the file rather // than surfacing an unrecoverable error, because the index is derived data // and the key was never a user secret. const other = MailIndex.open({ storeDir, accountId, key: randomBytes(32) }); expect(other.search({ query: 'Zurich' })).toHaveLength(0); other.close(); }); it('refuses a key of the wrong length', () => { expect(() => MailIndex.open({ storeDir, accountId, key: randomBytes(16) })).toThrow(/32 bytes/); }); it('finds documents by body, title and people', () => { const index = open(); index.upsert([doc()]); expect(index.search({ query: 'Zurich' }).map((h) => h.id)).toEqual(['M1']); expect(index.search({ query: 'quarterly' }).map((h) => h.id)).toEqual(['M1']); expect(index.search({ query: 'sophie@example.com' }).map((h) => h.id)).toEqual(['M1']); expect(index.search({ query: 'nonexistentword' })).toHaveLength(0); index.close(); }); it('returns a snippet for use as LLM context', () => { const index = open(); index.upsert([doc()]); const [hit] = index.search({ query: 'Zurich' }); expect(hit.snippet).toContain('[Zurich]'); expect(hit.metadata.threadId).toBe('T1'); index.close(); }); it('upserting the same id REPLACES the FTS row rather than duplicating it', () => { const index = open(); index.upsert([doc()]); index.upsert([doc({ body: 'Completely different content about Geneva.' })]); // One row, and the OLD text must no longer match - the classic // stale-FTS-row bug when the index is maintained by hand. expect(index.search({ query: 'Geneva' })).toHaveLength(1); expect(index.search({ query: 'Zurich' })).toHaveLength(0); expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(1); index.close(); }); it('scopes rows by JMAP account, so delegated accounts cannot merge', () => { const index = open(); index.upsert([ doc({ jmapAccountId: 'acc1', id: 'X', body: 'shared secret alpha' }), // Same JMAP id under a different account - legal, since JMAP ids are only // unique within an account (see namespaceMailboxIds in lib/jmap/client.ts). doc({ jmapAccountId: 'acc2', id: 'X', body: 'shared secret beta' }), ]); expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(2); const hits = index.search({ query: 'secret' }); expect(hits).toHaveLength(2); expect(new Set(hits.map((h) => h.jmapAccountId))).toEqual(new Set(['acc1', 'acc2'])); index.close(); }); it('filters by content type and searches across all four by default', () => { const index = open(); index.upsert([ doc({ contentType: 'mail', id: 'm', title: 'Zurich mail' }), doc({ contentType: 'calendar', id: 'c', title: 'Zurich meeting' }), doc({ contentType: 'contact', id: 'k', title: 'Zurich person', occurredAt: null }), doc({ contentType: 'file', id: 'f', title: 'Zurich file' }), ]); expect(index.search({ query: 'Zurich' })).toHaveLength(4); expect(index.search({ query: 'Zurich', types: ['calendar'] }).map((h) => h.id)).toEqual(['c']); expect(new Set(index.search({ query: 'Zurich', types: ['mail', 'file'] }).map((h) => h.id))) .toEqual(new Set(['m', 'f'])); index.close(); }); it('weights a title hit above a body-only hit', () => { const index = open(); index.upsert([ doc({ id: 'body-only', title: 'unrelated', body: 'mentions lease once' }), doc({ id: 'in-title', title: 'lease renewal', body: 'unrelated text' }), ]); // bm25 is negative and lower is better, so the title hit must come first. expect(index.search({ query: 'lease' })[0].id).toBe('in-title'); index.close(); }); it('removes documents and their FTS rows', () => { const index = open(); index.upsert([doc()]); expect(index.remove('acc1', 'mail', ['M1'])).toBe(1); expect(index.search({ query: 'Zurich' })).toHaveLength(0); expect(index.remove('acc1', 'mail', ['does-not-exist'])).toBe(0); index.close(); }); it('prunes by date without touching newer rows', () => { const index = open(); index.upsert([ doc({ id: 'old', occurredAt: '2020-01-01T00:00:00Z', body: 'ancient lease' }), doc({ id: 'new', occurredAt: '2026-08-01T00:00:00Z', body: 'current lease' }), ]); expect(index.pruneOlderThan('acc1', 'mail', '2026-01-01T00:00:00Z')).toBe(1); expect(index.search({ query: 'lease' }).map((h) => h.id)).toEqual(['new']); index.close(); }); it('reports existing ids and per-type stats', () => { const index = open(); index.upsert([doc({ id: 'a' }), doc({ id: 'b' }), doc({ contentType: 'file', id: 'f' })]); expect(index.existingIds('acc1', 'mail')).toEqual(new Set(['a', 'b'])); const stats = index.stats(); expect(stats.find((s) => s.contentType === 'mail')?.count).toBe(2); expect(stats.find((s) => s.contentType === 'file')?.count).toBe(1); index.close(); }); it('survives reopening and keeps the data', () => { const first = open(); first.upsert([doc()]); first.close(); const second = open(); expect(second.search({ query: 'Zurich' })).toHaveLength(1); second.close(); }); it('tolerates a hostile query string end to end', () => { const index = open(); index.upsert([doc()]); for (const q of ['"', '*', 'a" OR "b', 'NEAR(', ')', 'AND', '^', ':', '-']) { expect(() => index.search({ query: q })).not.toThrow(); } index.close(); }); });