SRCmail/lib/mail-index/extract.ts

// PURE JMAP-object -> IndexDoc extractors.
//
// Deliberately free of database, network and store access so every shape
// decision here is unit-testable on its own. The JMAP shapes are awkward
// enough (JSContact keyed maps, JSCalendar participants, FileNode's `modified`
// rather than `updated`) that this is where the bugs would otherwise hide.

import type { Email, CalendarEvent, ContactCard, FileNode, EmailAddress } from '@/lib/jmap/types';
import type { IndexDoc } from './store';

/** Hard cap on indexed body text per document. Keeps one enormous mail from dominating the file. */
export const MAX_BODY_CHARS = 32_000;

/**
 * Minimal HTML -> text, for mail that has no `text/plain` alternative.
 *
 * Not a sanitiser and not trying to be: this output is never rendered, only
 * tokenised by FTS5 and possibly handed to an LLM as context. The repo's
 * `dompurify` needs a DOM and this runs in Node, so a DOM-free reduction is the
 * right tool. Order matters - script/style content must go before tags are
 * stripped, or their contents would leak into the index as searchable text.
 */
export function htmlToText(html: string): string {
  return html
    .replace(/<!--[\s\S]*?-->/g, ' ')
    .replace(/<(script|style|head)\b[\s\S]*?<\/\1>/gi, ' ')
    .replace(/<br\s*\/?>/gi, '\n')
    .replace(/<\/(p|div|tr|li|h[1-6]|blockquote)>/gi, '\n')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/gi, ' ')
    .replace(/&amp;/gi, '&')
    .replace(/&lt;/gi, '<')
    .replace(/&gt;/gi, '>')
    .replace(/&quot;/gi, '"')
    .replace(/&#(\d+);/g, (_m, d: string) => {
      const code = Number(d);
      return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
    })
    .replace(/&#x([0-9a-f]+);/gi, (_m, h: string) => {
      const code = parseInt(h, 16);
      return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
    })
    .replace(/[ \t\u00a0]+/g, ' ')
    .replace(/\s*\n\s*/g, '\n')
    .trim();
}

export function normaliseText(s: string | null | undefined): string {
  if (!s) return '';
  return s.replace(/\r\n?/g, '\n').replace(/[ \t\u00a0]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
}

function clamp(s: string, max = MAX_BODY_CHARS): string {
  return s.length <= max ? s : s.slice(0, max);
}

function formatAddresses(list: readonly EmailAddress[] | undefined): string {
  if (!list || list.length === 0) return '';
  return list
    .map((a) => [a.name, a.email].filter((p) => typeof p === 'string' && p.length > 0).join(' '))
    .filter((s) => s.length > 0)
    .join(', ');
}

/** Values of a JSContact/JSCalendar keyed map, in a stable order. */
function mapValues<T>(m: Record<string, T> | null | undefined): T[] {
  if (!m || typeof m !== 'object') return [];
  return Object.keys(m).sort().map((k) => m[k]);
}

function joinUnique(parts: Array<string | undefined | null>): string {
  const seen = new Set<string>();
  const out: string[] = [];
  for (const p of parts) {
    const v = typeof p === 'string' ? p.trim() : '';
    if (!v || seen.has(v)) continue;
    seen.add(v);
    out.push(v);
  }
  return out.join(', ');
}

// ── mail ────────────────────────────────────────────────────────────────────

/**
 * Resolves an Email's plain-text body from `bodyValues`, preferring the
 * `text/plain` alternative and falling back to flattening the HTML one.
 *
 * `textBody`/`htmlBody` reference parts by `partId`; the text itself only
 * arrives in `bodyValues` when the `Email/get` asked for it
 * (`fetchTextBodyValues` / `fetchHTMLBodyValues`). A caller that forgets that
 * gets an empty body rather than an error, which is exactly the kind of silent
 * hole worth naming here.
 */
export function emailBodyText(email: Email): string {
  const values = email.bodyValues ?? {};
  const fromParts = (parts: typeof email.textBody): string =>
    (parts ?? [])
      .map((p) => values[p.partId]?.value ?? '')
      .filter((v) => v.length > 0)
      .join('\n\n');

  const plain = fromParts(email.textBody);
  if (plain.trim().length > 0) return normaliseText(plain);

  const html = fromParts(email.htmlBody);
  if (html.trim().length > 0) return normaliseText(htmlToText(html));

  // Last resort: the server-computed preview. Better than nothing for a search
  // index, and it costs no extra round trip.
  return normaliseText(email.preview);
}

export function extractMail(jmapAccountId: string, email: Email): IndexDoc {
  const body = clamp(emailBodyText(email));
  return {
    jmapAccountId,
    contentType: 'mail',
    id: email.id,
    title: normaliseText(email.subject) || '(no subject)',
    people: joinUnique([
      formatAddresses(email.from),
      formatAddresses(email.to),
      formatAddresses(email.cc),
    ]),
    body,
    occurredAt: email.receivedAt ?? null,
    metadata: {
      threadId: email.threadId,
      from: email.from?.[0]?.email ?? null,
      fromName: email.from?.[0]?.name ?? null,
      hasAttachment: !!email.hasAttachment,
      size: email.size ?? null,
      mailboxIds: Object.keys(email.mailboxIds ?? {}),
      preview: normaliseText(email.preview).slice(0, 300),
    },
  };
}

// ── calendar ────────────────────────────────────────────────────────────────

export function extractCalendarEvent(jmapAccountId: string, event: CalendarEvent): IndexDoc {
  const participants = mapValues(event.participants);
  const participantText = joinUnique(
    participants.flatMap((p) => [
      p?.name,
      p?.email,
      p?.calendarAddress?.replace(/^mailto:/i, ''),
      ...Object.values(p?.sendTo ?? {}).map((v) =>
        typeof v === 'string' ? v.replace(/^mailto:/i, '') : '',
      ),
    ]),
  );

  const locations = mapValues(event.locations)
    .map((l) => normaliseText(l?.name))
    .filter((s) => s.length > 0);

  // `descriptionContentType` can legitimately be text/html.
  const rawDescription = normaliseText(event.description);
  const description = /html/i.test(event.descriptionContentType ?? '')
    ? normaliseText(htmlToText(rawDescription))
    : rawDescription;

  const keywords = Object.keys(event.keywords ?? {});
  const categories = Object.keys(event.categories ?? {});

  return {
    jmapAccountId,
    contentType: 'calendar',
    id: event.id,
    title: normaliseText(event.title) || '(untitled event)',
    people: joinUnique([event.organizerCalendarAddress?.replace(/^mailto:/i, ''), participantText]),
    body: clamp(
      [description, locations.join(', '), keywords.join(' '), categories.join(' ')]
        .filter((s) => s.length > 0)
        .join('\n\n'),
    ),
    // `utcStart` is the resolved instant the app computes; `start` is local
    // wall-clock without a zone, so prefer utcStart for ordering.
    occurredAt: event.utcStart ?? event.start ?? null,
    metadata: {
      start: event.start ?? null,
      utcStart: event.utcStart ?? null,
      utcEnd: event.utcEnd ?? null,
      timeZone: event.timeZone ?? null,
      showWithoutTime: !!event.showWithoutTime,
      status: event.status ?? null,
      locations,
      calendarIds: Object.keys(event.calendarIds ?? {}),
      participantCount: participants.length,
    },
  };
}

// ── contacts ────────────────────────────────────────────────────────────────

export function contactDisplayName(card: ContactCard): string {
  const full = normaliseText(card.name?.full);
  if (full) return full;
  const components = card.name?.components ?? [];
  const ordered = ['prefix', 'given', 'given2', 'additional', 'middle', 'surname', 'surname2', 'suffix'];
  const byKind = components
    .slice()
    .sort((a, b) => ordered.indexOf(a.kind) - ordered.indexOf(b.kind))
    .map((c) => c.value)
    .filter((v) => typeof v === 'string' && v.trim().length > 0)
    .join(' ');
  if (byKind.trim()) return normaliseText(byKind);
  const firstEmail = mapValues(card.emails)[0]?.address;
  if (firstEmail) return firstEmail;
  const org = mapValues(card.organizations)[0]?.name;
  return normaliseText(org) || '(unnamed contact)';
}

export function extractContact(jmapAccountId: string, card: ContactCard): IndexDoc {
  const emails = mapValues(card.emails).map((e) => e.address).filter(Boolean);
  const phones = mapValues(card.phones).map((p) => p.number).filter(Boolean);
  const nicknames = mapValues(card.nicknames)
    .map((n) => n?.name)
    .filter((v): v is string => typeof v === 'string' && v.length > 0);
  const orgs = mapValues(card.organizations).map((o) => o.name).filter((v): v is string => !!v);
  const titles = mapValues(card.titles).map((t) => t.name).filter(Boolean);
  const notes = mapValues(card.notes).map((n) => n.note).filter(Boolean);
  // `full` (RFC 9553) when present, else the legacy flat fields vCard import
  // produces, else the ordered components. All three shapes occur in this type.
  const addresses = mapValues(card.addresses)
    .map((a) =>
      normaliseText(
        a?.full ||
          [a?.street, a?.locality, a?.region, a?.postcode, a?.country]
            .filter((p): p is string => typeof p === 'string' && p.length > 0)
            .join(', ') ||
          (a?.components ?? []).map((c) => c.value).join(' '),
      ),
    )
    .filter((s) => s.length > 0);

  return {
    jmapAccountId,
    contentType: 'contact',
    id: card.id,
    title: contactDisplayName(card),
    // Emails/phones go in `people` (weighted above body) because "who is
    // this / what's their number" is the dominant contact lookup.
    people: joinUnique([...emails, ...phones, ...nicknames]),
    body: clamp([...orgs, ...titles, ...addresses, ...notes].filter(Boolean).join('\n')),
    // A contact has no meaningful single date; JSContact `updated` is optional
    // and not on this repo's type, so leave it null and rank by relevance only.
    occurredAt: null,
    metadata: {
      kind: card.kind ?? null,
      emails,
      phones,
      organizations: orgs,
      addressBookIds: Object.keys(card.addressBookIds ?? {}),
    },
  };
}

// ── files ───────────────────────────────────────────────────────────────────

/**
 * METADATA ONLY - filename, path, dates, size, owner. Deliberately NOT file
 * content: extracting searchable text from arbitrary PDFs / office documents /
 * images is a materially bigger problem (per-format parsers, OCR, size limits,
 * untrusted-input parsing in a process holding the user's mail) and is a
 * separate piece of work. `path` is passed in by the caller because a FileNode
 * only knows its `parentId`; resolving the chain is the caller's job.
 */
export function extractFile(
  jmapAccountId: string,
  node: FileNode,
  opts: { path?: string; ownerName?: string } = {},
): IndexDoc {
  const dirPath = normaliseText(opts.path);
  const isDirectory = node.type === 'd';
  return {
    jmapAccountId,
    contentType: 'file',
    id: node.id,
    title: normaliseText(node.name) || '(unnamed file)',
    people: joinUnique([opts.ownerName, node.accountName]),
    // The path is genuinely searchable text ("that thing in Invoices/2026"),
    // and the extension is worth tokenising on its own.
    body: clamp(
      [dirPath, isDirectory ? 'folder' : node.type, fileExtension(node.name)]
        .filter((s) => s && s.length > 0)
        .join('\n'),
    ),
    // FileNode has `modified`, NOT `updated` - asking for the wrong name
    // silently yields undefined (this repo hit that as #700).
    occurredAt: node.modified ?? node.created ?? null,
    metadata: {
      path: dirPath || null,
      mimeType: isDirectory ? null : node.type,
      isDirectory,
      size: typeof node.size === 'number' ? node.size : null,
      created: node.created ?? null,
      modified: node.modified ?? null,
      parentId: node.parentId ?? null,
      contentIndexed: false,
    },
  };
}

function fileExtension(name: string | undefined): string {
  if (!name) return '';
  const i = name.lastIndexOf('.');
  return i > 0 && i < name.length - 1 ? name.slice(i + 1).toLowerCase() : '';
}