diff --git a/lib/mail-index/__tests__/extract.test.ts b/lib/mail-index/__tests__/extract.test.ts new file mode 100644 index 00000000..11317fed --- /dev/null +++ b/lib/mail-index/__tests__/extract.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from 'vitest'; +import type { + CalendarEvent, CalendarParticipant, ContactCard, Email, EmailBodyPart, FileNode, +} from '@/lib/jmap/types'; +import { + contactDisplayName, emailBodyText, extractCalendarEvent, extractContact, extractFile, + extractMail, htmlToText, MAX_BODY_CHARS, normaliseText, +} from '../extract'; +import { buildFilePaths } from '../jmap'; + +describe('htmlToText', () => { + it('drops script and style CONTENT, not just the tags', () => { + // The important case: a naive `<[^>]+>` strip leaves the script body behind + // as searchable text, so a page full of JS would pollute the index. + const out = htmlToText('

Hello

'); + expect(out).toContain('Hello'); + expect(out).not.toContain('secretToken'); + expect(out).not.toContain('abc123'); + expect(out).not.toContain('color:red'); + }); + + it('turns block boundaries into newlines and decodes entities', () => { + expect(htmlToText('

one

two

')).toBe('one\ntwo'); + expect(htmlToText('a
b')).toBe('a\nb'); + expect(htmlToText('R&D <tag> "q"  x')).toBe('R&D "q" x'); + expect(htmlToText('€10 €20')).toBe('€10 €20'); + }); + + it('ignores comments and out-of-range numeric entities without throwing', () => { + expect(htmlToText('ab')).toBe('a b'); + expect(() => htmlToText('� �')).not.toThrow(); + }); +}); + +describe('normaliseText', () => { + it('collapses runs of spaces, tabs and non-breaking spaces', () => { + expect(normaliseText('a \t   b')).toBe('a b'); + }); + it('caps blank-line runs and handles null/undefined', () => { + expect(normaliseText('a\n\n\n\n\nb')).toBe('a\n\nb'); + expect(normaliseText(undefined)).toBe(''); + expect(normaliseText(null)).toBe(''); + }); +}); + +function baseEmail(overrides: Partial = {}): Email { + return { + id: 'M1', threadId: 'T1', mailboxIds: { mb1: true }, keywords: {}, + size: 100, receivedAt: '2026-08-01T10:00:00Z', hasAttachment: false, + ...overrides, + } as Email; +} + +describe('emailBodyText', () => { + it('prefers the text/plain part', () => { + const email = baseEmail({ + textBody: [{ partId: 'p1' } as EmailBodyPart], + htmlBody: [{ partId: 'p2' } as EmailBodyPart], + bodyValues: { p1: { value: 'plain wins' }, p2: { value: 'html loses' } }, + }); + expect(emailBodyText(email)).toBe('plain wins'); + }); + + it('falls back to flattened HTML when there is no plain alternative', () => { + const email = baseEmail({ + htmlBody: [{ partId: 'p2' } as EmailBodyPart], + bodyValues: { p2: { value: '

hello

world

' } }, + }); + expect(emailBodyText(email)).toBe('hello\nworld'); + }); + + it('falls back to preview when bodyValues is missing entirely', () => { + // This is the shape a caller gets when the Email/get omitted + // fetchTextBodyValues - a silent empty body if we did not handle it. + const email = baseEmail({ + textBody: [{ partId: 'p1' } as EmailBodyPart], + preview: 'server preview text', + }); + expect(emailBodyText(email)).toBe('server preview text'); + }); + + it('treats a whitespace-only plain part as absent', () => { + const email = baseEmail({ + textBody: [{ partId: 'p1' } as EmailBodyPart], + htmlBody: [{ partId: 'p2' } as EmailBodyPart], + bodyValues: { p1: { value: ' \n ' }, p2: { value: 'real content' } }, + }); + expect(emailBodyText(email)).toBe('real content'); + }); +}); + +describe('extractMail', () => { + it('flattens addresses into `people` and keeps metadata', () => { + const doc = extractMail('acc1', baseEmail({ + subject: 'Quarterly budget', + from: [{ name: 'Sophie Müller', email: 'sophie@example.com' }], + to: [{ email: 'me@example.com' }], + cc: [{ name: 'Bob', email: 'bob@example.com' }], + preview: 'hi', + })); + expect(doc.contentType).toBe('mail'); + expect(doc.title).toBe('Quarterly budget'); + expect(doc.people).toContain('Sophie Müller sophie@example.com'); + expect(doc.people).toContain('bob@example.com'); + expect(doc.occurredAt).toBe('2026-08-01T10:00:00Z'); + expect(doc.metadata.threadId).toBe('T1'); + expect(doc.metadata.mailboxIds).toEqual(['mb1']); + }); + + it('substitutes a placeholder title rather than indexing an empty one', () => { + expect(extractMail('acc1', baseEmail()).title).toBe('(no subject)'); + }); + + it('clamps a huge body', () => { + const doc = extractMail('acc1', baseEmail({ + textBody: [{ partId: 'p1' } as EmailBodyPart], + bodyValues: { p1: { value: 'x'.repeat(MAX_BODY_CHARS * 2) } }, + })); + expect(doc.body.length).toBe(MAX_BODY_CHARS); + }); +}); + +function baseEvent(overrides: Partial = {}): CalendarEvent { + return { + id: 'E1', calendarIds: { c1: true }, isDraft: false, isOrigin: true, + utcStart: '2026-08-10T09:00:00Z', utcEnd: '2026-08-10T10:00:00Z', + '@type': 'Event', uid: 'u1', title: 'Standup', description: '', + descriptionContentType: 'text/plain', created: null, updated: '2026-08-01T00:00:00Z', + sequence: 0, start: '2026-08-10T11:00:00', duration: 'PT1H', timeZone: 'Europe/Zurich', + showWithoutTime: false, status: 'confirmed', freeBusyStatus: 'busy', privacy: 'public', + color: null, keywords: null, categories: null, locale: null, replyTo: null, + organizerCalendarAddress: null, participants: null, mayInviteSelf: false, + mayInviteOthers: false, hideAttendees: false, recurrenceId: null, + recurrenceIdTimeZone: null, recurrenceRules: null, recurrenceOverrides: null, + excludedRecurrenceRules: null, useDefaultAlerts: false, alerts: null, + locations: null, virtualLocations: null, links: null, relatedTo: null, + ...overrides, + } as CalendarEvent; +} + +describe('extractCalendarEvent', () => { + it('indexes description, location, attendees and organizer', () => { + const doc = extractCalendarEvent('acc1', baseEvent({ + title: 'Lease decision', + description: 'Zurich office lease renewal', + locations: { l1: { '@type': 'Location', name: 'Room 3.14', description: null, locationTypes: null, coordinates: null, timeZone: null, links: null, relativeTo: null } }, + organizerCalendarAddress: 'mailto:boss@example.com', + // A partial participant on purpose: servers omit most JSCalendar fields, + // and the extractor must cope with exactly this shape. + participants: { + p1: { name: 'Ana', email: 'ana@example.com', sendTo: { imip: 'mailto:ana@example.com' } } as unknown as CalendarParticipant, + }, + })); + expect(doc.title).toBe('Lease decision'); + expect(doc.body).toContain('Zurich office lease renewal'); + expect(doc.body).toContain('Room 3.14'); + // mailto: prefixes stripped so the address tokenises like every other one. + expect(doc.people).toContain('boss@example.com'); + expect(doc.people).not.toContain('mailto:'); + expect(doc.people).toContain('ana@example.com'); + expect(doc.metadata.participantCount).toBe(1); + }); + + it('flattens an HTML description', () => { + const doc = extractCalendarEvent('acc1', baseEvent({ + description: '

agenda

', + descriptionContentType: 'text/html', + })); + expect(doc.body).toContain('agenda'); + expect(doc.body).not.toContain('bad()'); + }); + + it('prefers utcStart over the zone-less local start for ordering', () => { + expect(extractCalendarEvent('acc1', baseEvent()).occurredAt).toBe('2026-08-10T09:00:00Z'); + expect(extractCalendarEvent('acc1', baseEvent({ utcStart: null })).occurredAt) + .toBe('2026-08-10T11:00:00'); + }); +}); + +describe('extractContact', () => { + const card = (overrides: Partial = {}): ContactCard => + ({ id: 'C1', addressBookIds: { a1: true }, ...overrides }) as ContactCard; + + it('uses name.full when present', () => { + expect(contactDisplayName(card({ name: { full: 'Ada Lovelace' } }))).toBe('Ada Lovelace'); + }); + + it('assembles components in the right order when full is absent', () => { + expect(contactDisplayName(card({ + name: { components: [{ kind: 'surname', value: 'Hopper' }, { kind: 'given', value: 'Grace' }] }, + }))).toBe('Grace Hopper'); + }); + + it('degrades to an email, then an org, then a placeholder', () => { + expect(contactDisplayName(card({ emails: { e: { address: 'x@y.z' } } }))).toBe('x@y.z'); + expect(contactDisplayName(card({ organizations: { o: { name: 'ACME' } } }))).toBe('ACME'); + expect(contactDisplayName(card())).toBe('(unnamed contact)'); + }); + + it('puts emails and phones in `people` and notes/orgs in `body`', () => { + const doc = extractContact('acc1', card({ + name: { full: 'Ada Lovelace' }, + emails: { e1: { address: 'ada@example.com' } }, + phones: { p1: { number: '+41 44 000 00 00' } }, + organizations: { o1: { name: 'Analytical Engines' } }, + notes: { n1: { note: 'met at the Zurich conference' } }, + nicknames: { k1: { name: 'The Countess' } }, + })); + expect(doc.people).toContain('ada@example.com'); + expect(doc.people).toContain('+41 44 000 00 00'); + expect(doc.people).toContain('The Countess'); + expect(doc.body).toContain('Analytical Engines'); + expect(doc.body).toContain('met at the Zurich conference'); + // A contact has no single meaningful date; ranking is relevance-only. + expect(doc.occurredAt).toBeNull(); + }); + + it('handles both RFC 9553 and legacy flat address shapes', () => { + expect(extractContact('acc1', card({ addresses: { a: { full: 'Bahnhofstrasse 1, Zurich' } } })).body) + .toContain('Bahnhofstrasse 1, Zurich'); + expect(extractContact('acc1', card({ addresses: { a: { street: 'Bahnhofstrasse 1', locality: 'Zurich' } } })).body) + .toContain('Bahnhofstrasse 1, Zurich'); + }); +}); + +describe('extractFile', () => { + const node = (overrides: Partial = {}): FileNode => + ({ + id: 'F1', parentId: null, name: 'invoice.pdf', type: 'application/pdf', + blobId: 'b1', size: 1234, created: '2026-07-01T00:00:00Z', + modified: '2026-07-15T00:00:00Z', ...overrides, + }) as FileNode; + + it('indexes metadata only and says so', () => { + const doc = extractFile('acc1', node(), { path: 'Finance/2026' }); + expect(doc.title).toBe('invoice.pdf'); + expect(doc.body).toContain('Finance/2026'); + expect(doc.body).toContain('pdf'); + expect(doc.metadata.contentIndexed).toBe(false); + expect(doc.metadata.mimeType).toBe('application/pdf'); + expect(doc.metadata.size).toBe(1234); + }); + + it('uses `modified` (FileNode has no `updated`) and falls back to `created`', () => { + expect(extractFile('acc1', node()).occurredAt).toBe('2026-07-15T00:00:00Z'); + expect(extractFile('acc1', node({ modified: undefined as unknown as string })).occurredAt) + .toBe('2026-07-01T00:00:00Z'); + }); + + it('marks directories', () => { + const doc = extractFile('acc1', node({ name: 'Finance', type: 'd', blobId: null })); + expect(doc.metadata.isDirectory).toBe(true); + expect(doc.metadata.mimeType).toBeNull(); + expect(doc.body).toContain('folder'); + }); +}); + +describe('buildFilePaths', () => { + it('resolves the PARENT chain, excluding the node itself', () => { + const nodes = [ + { id: 'root', parentId: null, name: 'Finance' }, + { id: 'year', parentId: 'root', name: '2026' }, + { id: 'file', parentId: 'year', name: 'invoice.pdf' }, + ] as FileNode[]; + const paths = buildFilePaths(nodes); + expect(paths.get('file')).toBe('Finance/2026'); + expect(paths.get('year')).toBe('Finance'); + expect(paths.get('root')).toBe(''); + }); + + it('truncates rather than failing when an ancestor is not in the set', () => { + const nodes = [{ id: 'file', parentId: 'missing', name: 'x.txt' }] as FileNode[]; + expect(buildFilePaths(nodes).get('file')).toBe(''); + }); + + it('terminates on a parent cycle', () => { + const nodes = [ + { id: 'a', parentId: 'b', name: 'A' }, + { id: 'b', parentId: 'a', name: 'B' }, + ] as FileNode[]; + expect(() => buildFilePaths(nodes)).not.toThrow(); + }); +}); diff --git a/lib/mail-index/__tests__/store.test.ts b/lib/mail-index/__tests__/store.test.ts new file mode 100644 index 00000000..610e03b6 --- /dev/null +++ b/lib/mail-index/__tests__/store.test.ts @@ -0,0 +1,261 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomBytes } from 'node:crypto'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { isSqlcipherAvailable } from '../binding'; +import { accountFileToken, getStoreDir, indexDbPath, STORE_DIR_ENV } from '../paths'; +import { MailIndex, toFtsMatchQuery, type IndexDoc } from '../store'; + +describe('toFtsMatchQuery', () => { + it('quotes every token so FTS5 operators in user input cannot break the query', () => { + // FTS5's MATCH grammar is NOT protected by SQL parameter binding: a bare + // quote or a stray NEAR/AND/* raises `fts5: syntax error`, which would turn + // a search box into a 500. + expect(toFtsMatchQuery('a" OR b')).toBe('"a" AND "OR" AND "b"'); + // No trailing `*` here: the final token is one character, below the + // prefix-match threshold (see the next test). + expect(toFtsMatchQuery('NEAR(x y)')).toBe('"NEAR" AND "x" AND "y"'); + expect(toFtsMatchQuery('NEAR(x yes)')).toBe('"NEAR" AND "x" AND "yes"*'); + expect(toFtsMatchQuery('foo*')).toBe('"foo"*'); + expect(toFtsMatchQuery('a AND NOT b')).toContain('"NOT"'); + }); + + it('prefix-matches only the final token, and only when it is long enough', () => { + expect(toFtsMatchQuery('zurich lea')).toBe('"zurich" AND "lea"*'); + // Two characters would match too much of a mailbox to be useful. + expect(toFtsMatchQuery('zurich le')).toBe('"zurich" AND "le"'); + }); + + it('keeps unicode letters, emails and hyphenated words', () => { + expect(toFtsMatchQuery('Müller')).toBe('"Müller"*'); + expect(toFtsMatchQuery('東京')).toBe('"東京"'); + expect(toFtsMatchQuery('a@b.com')).toBe('"a@b.com"*'); + expect(toFtsMatchQuery("O'Brien-Smith")).toBe('"O\'Brien-Smith"*'); + }); + + it('returns null for input with no usable tokens', () => { + expect(toFtsMatchQuery('')).toBeNull(); + expect(toFtsMatchQuery(' ')).toBeNull(); + expect(toFtsMatchQuery('***')).toBeNull(); + expect(toFtsMatchQuery(undefined as unknown as string)).toBeNull(); + }); + + it('bounds the token count', () => { + const many = Array.from({ length: 100 }, (_, i) => `w${i}`).join(' '); + expect((toFtsMatchQuery(many) ?? '').split(' AND ')).toHaveLength(24); + }); +}); + +describe('paths', () => { + const original = process.env[STORE_DIR_ENV]; + afterEach(() => { + if (original === undefined) delete process.env[STORE_DIR_ENV]; + else process.env[STORE_DIR_ENV] = original; + }); + + it('is disabled unless the env var is set - the hosted-deployment gate', () => { + delete process.env[STORE_DIR_ENV]; + expect(getStoreDir()).toBeNull(); + process.env[STORE_DIR_ENV] = ''; + expect(getStoreDir()).toBeNull(); + }); + + it('rejects a relative path, which would resolve against the server cwd', () => { + process.env[STORE_DIR_ENV] = 'offline'; + expect(getStoreDir()).toBeNull(); + process.env[STORE_DIR_ENV] = '/abs/offline'; + expect(getStoreDir()).toBe('/abs/offline'); + }); + + it('hashes the filename so the directory is not an account inventory', () => { + const token = accountFileToken('linus@example.com'); + expect(token).toMatch(/^[0-9a-f]{32}$/); + expect(token).not.toContain('linus'); + expect(indexDbPath('/s', 'linus@example.com')).toBe(`/s/index/${token}.db`); + // Deterministic - the same account must resolve to the same file forever. + expect(accountFileToken('linus@example.com')).toBe(token); + }); +}); + +function doc(overrides: Partial = {}): IndexDoc { + return { + jmapAccountId: 'acc1', + contentType: 'mail', + id: 'M1', + title: 'Quarterly budget review', + people: 'Sophie Müller sophie@example.com', + body: 'The Zurich office lease renewal needs a decision before September.', + occurredAt: '2026-08-01T10:00:00Z', + metadata: { threadId: 'T1' }, + ...overrides, + }; +} + +// The native binding is an OPTIONAL dependency, so these skip rather than fail +// on a platform with no prebuild (e.g. Alpine/musl in CI containers). +describe.skipIf(!isSqlcipherAvailable())('MailIndex (real SQLCipher)', () => { + let storeDir: string; + const accountId = 'linus@example.com'; + const key = randomBytes(32); + + beforeEach(() => { + storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mail-index-test-')); + }); + afterEach(() => { + fs.rmSync(storeDir, { recursive: true, force: true }); + }); + + const open = () => MailIndex.open({ storeDir, accountId, key }); + + it('writes an ENCRYPTED file - no plaintext recoverable from the raw bytes', () => { + const index = open(); + index.upsert([doc()]); + index.close(); + + const bytes = fs.readFileSync(indexDbPath(storeDir, accountId)); + // The canary check, not just a header check: this is the assertion that + // would have caught `PRAGMA key` being a silent no-op. + expect(bytes.includes('Zurich office lease')).toBe(false); + expect(bytes.includes('Quarterly budget')).toBe(false); + expect(bytes.subarray(0, 15).toString('latin1')).not.toBe('SQLite format 3'); + }); + + it('rejects a wrong key and rebuilds instead of throwing at the caller', () => { + const index = open(); + index.upsert([doc()]); + index.close(); + + // A different key cannot read the data; the store recreates the file rather + // than surfacing an unrecoverable error, because the index is derived data + // and the key was never a user secret. + const other = MailIndex.open({ storeDir, accountId, key: randomBytes(32) }); + expect(other.search({ query: 'Zurich' })).toHaveLength(0); + other.close(); + }); + + it('refuses a key of the wrong length', () => { + expect(() => MailIndex.open({ storeDir, accountId, key: randomBytes(16) })).toThrow(/32 bytes/); + }); + + it('finds documents by body, title and people', () => { + const index = open(); + index.upsert([doc()]); + expect(index.search({ query: 'Zurich' }).map((h) => h.id)).toEqual(['M1']); + expect(index.search({ query: 'quarterly' }).map((h) => h.id)).toEqual(['M1']); + expect(index.search({ query: 'sophie@example.com' }).map((h) => h.id)).toEqual(['M1']); + expect(index.search({ query: 'nonexistentword' })).toHaveLength(0); + index.close(); + }); + + it('returns a snippet for use as LLM context', () => { + const index = open(); + index.upsert([doc()]); + const [hit] = index.search({ query: 'Zurich' }); + expect(hit.snippet).toContain('[Zurich]'); + expect(hit.metadata.threadId).toBe('T1'); + index.close(); + }); + + it('upserting the same id REPLACES the FTS row rather than duplicating it', () => { + const index = open(); + index.upsert([doc()]); + index.upsert([doc({ body: 'Completely different content about Geneva.' })]); + + // One row, and the OLD text must no longer match - the classic + // stale-FTS-row bug when the index is maintained by hand. + expect(index.search({ query: 'Geneva' })).toHaveLength(1); + expect(index.search({ query: 'Zurich' })).toHaveLength(0); + expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(1); + index.close(); + }); + + it('scopes rows by JMAP account, so delegated accounts cannot merge', () => { + const index = open(); + index.upsert([ + doc({ jmapAccountId: 'acc1', id: 'X', body: 'shared secret alpha' }), + // Same JMAP id under a different account - legal, since JMAP ids are only + // unique within an account (see namespaceMailboxIds in lib/jmap/client.ts). + doc({ jmapAccountId: 'acc2', id: 'X', body: 'shared secret beta' }), + ]); + expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(2); + const hits = index.search({ query: 'secret' }); + expect(hits).toHaveLength(2); + expect(new Set(hits.map((h) => h.jmapAccountId))).toEqual(new Set(['acc1', 'acc2'])); + index.close(); + }); + + it('filters by content type and searches across all four by default', () => { + const index = open(); + index.upsert([ + doc({ contentType: 'mail', id: 'm', title: 'Zurich mail' }), + doc({ contentType: 'calendar', id: 'c', title: 'Zurich meeting' }), + doc({ contentType: 'contact', id: 'k', title: 'Zurich person', occurredAt: null }), + doc({ contentType: 'file', id: 'f', title: 'Zurich file' }), + ]); + expect(index.search({ query: 'Zurich' })).toHaveLength(4); + expect(index.search({ query: 'Zurich', types: ['calendar'] }).map((h) => h.id)).toEqual(['c']); + expect(new Set(index.search({ query: 'Zurich', types: ['mail', 'file'] }).map((h) => h.id))) + .toEqual(new Set(['m', 'f'])); + index.close(); + }); + + it('weights a title hit above a body-only hit', () => { + const index = open(); + index.upsert([ + doc({ id: 'body-only', title: 'unrelated', body: 'mentions lease once' }), + doc({ id: 'in-title', title: 'lease renewal', body: 'unrelated text' }), + ]); + // bm25 is negative and lower is better, so the title hit must come first. + expect(index.search({ query: 'lease' })[0].id).toBe('in-title'); + index.close(); + }); + + it('removes documents and their FTS rows', () => { + const index = open(); + index.upsert([doc()]); + expect(index.remove('acc1', 'mail', ['M1'])).toBe(1); + expect(index.search({ query: 'Zurich' })).toHaveLength(0); + expect(index.remove('acc1', 'mail', ['does-not-exist'])).toBe(0); + index.close(); + }); + + it('prunes by date without touching newer rows', () => { + const index = open(); + index.upsert([ + doc({ id: 'old', occurredAt: '2020-01-01T00:00:00Z', body: 'ancient lease' }), + doc({ id: 'new', occurredAt: '2026-08-01T00:00:00Z', body: 'current lease' }), + ]); + expect(index.pruneOlderThan('acc1', 'mail', '2026-01-01T00:00:00Z')).toBe(1); + expect(index.search({ query: 'lease' }).map((h) => h.id)).toEqual(['new']); + index.close(); + }); + + it('reports existing ids and per-type stats', () => { + const index = open(); + index.upsert([doc({ id: 'a' }), doc({ id: 'b' }), doc({ contentType: 'file', id: 'f' })]); + expect(index.existingIds('acc1', 'mail')).toEqual(new Set(['a', 'b'])); + const stats = index.stats(); + expect(stats.find((s) => s.contentType === 'mail')?.count).toBe(2); + expect(stats.find((s) => s.contentType === 'file')?.count).toBe(1); + index.close(); + }); + + it('survives reopening and keeps the data', () => { + const first = open(); + first.upsert([doc()]); + first.close(); + const second = open(); + expect(second.search({ query: 'Zurich' })).toHaveLength(1); + second.close(); + }); + + it('tolerates a hostile query string end to end', () => { + const index = open(); + index.upsert([doc()]); + for (const q of ['"', '*', 'a" OR "b', 'NEAR(', ')', 'AND', '^', ':', '-']) { + expect(() => index.search({ query: q })).not.toThrow(); + } + index.close(); + }); +});