// PURE JMAP-object -> IndexDoc extractors. // // Deliberately free of database, network and store access so every shape // decision here is unit-testable on its own. The JMAP shapes are awkward // enough (JSContact keyed maps, JSCalendar participants, FileNode's `modified` // rather than `updated`) that this is where the bugs would otherwise hide. import type { Email, CalendarEvent, ContactCard, FileNode, EmailAddress } from '@/lib/jmap/types'; import type { IndexDoc } from './store'; /** Hard cap on indexed body text per document. Keeps one enormous mail from dominating the file. */ export const MAX_BODY_CHARS = 32_000; /** * Minimal HTML -> text, for mail that has no `text/plain` alternative. * * Not a sanitiser and not trying to be: this output is never rendered, only * tokenised by FTS5 and possibly handed to an LLM as context. The repo's * `dompurify` needs a DOM and this runs in Node, so a DOM-free reduction is the * right tool. Order matters - script/style content must go before tags are * stripped, or their contents would leak into the index as searchable text. */ export function htmlToText(html: string): string { return html .replace(//g, ' ') .replace(/<(script|style|head)\b[\s\S]*?<\/\1>/gi, ' ') .replace(//gi, '\n') .replace(/<\/(p|div|tr|li|h[1-6]|blockquote)>/gi, '\n') .replace(/<[^>]+>/g, ' ') .replace(/ /gi, ' ') .replace(/&/gi, '&') .replace(/</gi, '<') .replace(/>/gi, '>') .replace(/"/gi, '"') .replace(/&#(\d+);/g, (_m, d: string) => { const code = Number(d); return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' '; }) .replace(/&#x([0-9a-f]+);/gi, (_m, h: string) => { const code = parseInt(h, 16); return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' '; }) .replace(/[ \t\u00a0]+/g, ' ') .replace(/\s*\n\s*/g, '\n') .trim(); } export function normaliseText(s: string | null | undefined): string { if (!s) return ''; return s.replace(/\r\n?/g, '\n').replace(/[ \t\u00a0]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim(); } function clamp(s: string, max = MAX_BODY_CHARS): string { return s.length <= max ? s : s.slice(0, max); } function formatAddresses(list: readonly EmailAddress[] | undefined): string { if (!list || list.length === 0) return ''; return list .map((a) => [a.name, a.email].filter((p) => typeof p === 'string' && p.length > 0).join(' ')) .filter((s) => s.length > 0) .join(', '); } /** Values of a JSContact/JSCalendar keyed map, in a stable order. */ function mapValues(m: Record | null | undefined): T[] { if (!m || typeof m !== 'object') return []; return Object.keys(m).sort().map((k) => m[k]); } function joinUnique(parts: Array): string { const seen = new Set(); const out: string[] = []; for (const p of parts) { const v = typeof p === 'string' ? p.trim() : ''; if (!v || seen.has(v)) continue; seen.add(v); out.push(v); } return out.join(', '); } // ── mail ──────────────────────────────────────────────────────────────────── /** * Resolves an Email's plain-text body from `bodyValues`, preferring the * `text/plain` alternative and falling back to flattening the HTML one. * * `textBody`/`htmlBody` reference parts by `partId`; the text itself only * arrives in `bodyValues` when the `Email/get` asked for it * (`fetchTextBodyValues` / `fetchHTMLBodyValues`). A caller that forgets that * gets an empty body rather than an error, which is exactly the kind of silent * hole worth naming here. */ export function emailBodyText(email: Email): string { const values = email.bodyValues ?? {}; const fromParts = (parts: typeof email.textBody): string => (parts ?? []) .map((p) => values[p.partId]?.value ?? '') .filter((v) => v.length > 0) .join('\n\n'); const plain = fromParts(email.textBody); if (plain.trim().length > 0) return normaliseText(plain); const html = fromParts(email.htmlBody); if (html.trim().length > 0) return normaliseText(htmlToText(html)); // Last resort: the server-computed preview. Better than nothing for a search // index, and it costs no extra round trip. return normaliseText(email.preview); } export function extractMail(jmapAccountId: string, email: Email): IndexDoc { const body = clamp(emailBodyText(email)); return { jmapAccountId, contentType: 'mail', id: email.id, title: normaliseText(email.subject) || '(no subject)', people: joinUnique([ formatAddresses(email.from), formatAddresses(email.to), formatAddresses(email.cc), ]), body, occurredAt: email.receivedAt ?? null, metadata: { threadId: email.threadId, from: email.from?.[0]?.email ?? null, fromName: email.from?.[0]?.name ?? null, hasAttachment: !!email.hasAttachment, size: email.size ?? null, mailboxIds: Object.keys(email.mailboxIds ?? {}), preview: normaliseText(email.preview).slice(0, 300), }, }; } // ── calendar ──────────────────────────────────────────────────────────────── export function extractCalendarEvent(jmapAccountId: string, event: CalendarEvent): IndexDoc { const participants = mapValues(event.participants); const participantText = joinUnique( participants.flatMap((p) => [ p?.name, p?.email, p?.calendarAddress?.replace(/^mailto:/i, ''), ...Object.values(p?.sendTo ?? {}).map((v) => typeof v === 'string' ? v.replace(/^mailto:/i, '') : '', ), ]), ); const locations = mapValues(event.locations) .map((l) => normaliseText(l?.name)) .filter((s) => s.length > 0); // `descriptionContentType` can legitimately be text/html. const rawDescription = normaliseText(event.description); const description = /html/i.test(event.descriptionContentType ?? '') ? normaliseText(htmlToText(rawDescription)) : rawDescription; const keywords = Object.keys(event.keywords ?? {}); const categories = Object.keys(event.categories ?? {}); return { jmapAccountId, contentType: 'calendar', id: event.id, title: normaliseText(event.title) || '(untitled event)', people: joinUnique([event.organizerCalendarAddress?.replace(/^mailto:/i, ''), participantText]), body: clamp( [description, locations.join(', '), keywords.join(' '), categories.join(' ')] .filter((s) => s.length > 0) .join('\n\n'), ), // `utcStart` is the resolved instant the app computes; `start` is local // wall-clock without a zone, so prefer utcStart for ordering. occurredAt: event.utcStart ?? event.start ?? null, metadata: { start: event.start ?? null, utcStart: event.utcStart ?? null, utcEnd: event.utcEnd ?? null, timeZone: event.timeZone ?? null, showWithoutTime: !!event.showWithoutTime, status: event.status ?? null, locations, calendarIds: Object.keys(event.calendarIds ?? {}), participantCount: participants.length, }, }; } // ── contacts ──────────────────────────────────────────────────────────────── export function contactDisplayName(card: ContactCard): string { const full = normaliseText(card.name?.full); if (full) return full; const components = card.name?.components ?? []; const ordered = ['prefix', 'given', 'given2', 'additional', 'middle', 'surname', 'surname2', 'suffix']; const byKind = components .slice() .sort((a, b) => ordered.indexOf(a.kind) - ordered.indexOf(b.kind)) .map((c) => c.value) .filter((v) => typeof v === 'string' && v.trim().length > 0) .join(' '); if (byKind.trim()) return normaliseText(byKind); const firstEmail = mapValues(card.emails)[0]?.address; if (firstEmail) return firstEmail; const org = mapValues(card.organizations)[0]?.name; return normaliseText(org) || '(unnamed contact)'; } export function extractContact(jmapAccountId: string, card: ContactCard): IndexDoc { const emails = mapValues(card.emails).map((e) => e.address).filter(Boolean); const phones = mapValues(card.phones).map((p) => p.number).filter(Boolean); const nicknames = mapValues(card.nicknames) .map((n) => n?.name) .filter((v): v is string => typeof v === 'string' && v.length > 0); const orgs = mapValues(card.organizations).map((o) => o.name).filter((v): v is string => !!v); const titles = mapValues(card.titles).map((t) => t.name).filter(Boolean); const notes = mapValues(card.notes).map((n) => n.note).filter(Boolean); // `full` (RFC 9553) when present, else the legacy flat fields vCard import // produces, else the ordered components. All three shapes occur in this type. const addresses = mapValues(card.addresses) .map((a) => normaliseText( a?.full || [a?.street, a?.locality, a?.region, a?.postcode, a?.country] .filter((p): p is string => typeof p === 'string' && p.length > 0) .join(', ') || (a?.components ?? []).map((c) => c.value).join(' '), ), ) .filter((s) => s.length > 0); return { jmapAccountId, contentType: 'contact', id: card.id, title: contactDisplayName(card), // Emails/phones go in `people` (weighted above body) because "who is // this / what's their number" is the dominant contact lookup. people: joinUnique([...emails, ...phones, ...nicknames]), body: clamp([...orgs, ...titles, ...addresses, ...notes].filter(Boolean).join('\n')), // A contact has no meaningful single date; JSContact `updated` is optional // and not on this repo's type, so leave it null and rank by relevance only. occurredAt: null, metadata: { kind: card.kind ?? null, emails, phones, organizations: orgs, addressBookIds: Object.keys(card.addressBookIds ?? {}), }, }; } // ── files ─────────────────────────────────────────────────────────────────── /** * METADATA ONLY - filename, path, dates, size, owner. Deliberately NOT file * content: extracting searchable text from arbitrary PDFs / office documents / * images is a materially bigger problem (per-format parsers, OCR, size limits, * untrusted-input parsing in a process holding the user's mail) and is a * separate piece of work. `path` is passed in by the caller because a FileNode * only knows its `parentId`; resolving the chain is the caller's job. */ export function extractFile( jmapAccountId: string, node: FileNode, opts: { path?: string; ownerName?: string } = {}, ): IndexDoc { const dirPath = normaliseText(opts.path); const isDirectory = node.type === 'd'; return { jmapAccountId, contentType: 'file', id: node.id, title: normaliseText(node.name) || '(unnamed file)', people: joinUnique([opts.ownerName, node.accountName]), // The path is genuinely searchable text ("that thing in Invoices/2026"), // and the extension is worth tokenising on its own. body: clamp( [dirPath, isDirectory ? 'folder' : node.type, fileExtension(node.name)] .filter((s) => s && s.length > 0) .join('\n'), ), // FileNode has `modified`, NOT `updated` - asking for the wrong name // silently yields undefined (this repo hit that as #700). occurredAt: node.modified ?? node.created ?? null, metadata: { path: dirPath || null, mimeType: isDirectory ? null : node.type, isDirectory, size: typeof node.size === 'number' ? node.size : null, created: node.created ?? null, modified: node.modified ?? null, parentId: node.parentId ?? null, contentIndexed: false, }, }; } function fileExtension(name: string | undefined): string { if (!name) return ''; const i = name.lastIndexOf('.'); return i > 0 && i < name.length - 1 ? name.slice(i + 1).toLowerCase() : ''; }