SRCmail/lib/mail-index/store.ts

// The encrypted local search index: schema, open/close, upsert, search.
//
// One SQLite (SQLCipher) file per account. Rows are ALSO account-scoped
// internally - `(jmap_account_id, content_type, id)` - because a single login
// exposes the user's own JMAP account plus every delegated/shared account, and
// JMAP ids are unique only WITHIN an account (this codebase already works
// around that collision in `lib/jmap/client.ts:388`'s namespaceMailboxIds).
// One file per account keeps purge trivial; the composite key keeps
// delegated accounts from merging inside it.
//
// This is a SEARCH INDEX, not a mail replica. It is allowed to be stale, it is
// allowed to be incomplete, and it can be discarded and rebuilt at any time -
// which is why the schema-version mismatch path below simply drops everything
// rather than migrating.

import fs from 'node:fs';
import path from 'node:path';
import { loadSqlcipher, type SqlcipherDatabase } from './binding';
import { dbSiblings, indexDbPath } from './paths';

export const SCHEMA_VERSION = 1;

export type ContentType = 'mail' | 'calendar' | 'contact' | 'file';

export const CONTENT_TYPES: readonly ContentType[] = ['mail', 'calendar', 'contact', 'file'];

export function isContentType(v: unknown): v is ContentType {
  return typeof v === 'string' && (CONTENT_TYPES as readonly string[]).includes(v);
}

/**
 * One indexable thing, already flattened to text. Produced by the pure
 * extractors in `extract.ts` so that every JMAP-shape decision is unit-testable
 * without a database or a server.
 */
export interface IndexDoc {
  jmapAccountId: string;
  contentType: ContentType;
  /** JMAP id. Unique only within (jmapAccountId, contentType). */
  id: string;
  /** Subject / event title / contact display name / filename. */
  title: string;
  /** Addresses and names: sender+recipients, attendees, contact emails/phones, owner. */
  people: string;
  /** The bulk searchable text. Plain text only - never HTML. */
  body: string;
  /** ISO 8601, or null when the type has no meaningful date. Drives recency ordering. */
  occurredAt: string | null;
  /** Small type-specific extras returned verbatim to the caller (never searched). */
  metadata: Record<string, unknown>;
}

export interface SearchHit {
  contentType: ContentType;
  id: string;
  jmapAccountId: string;
  title: string;
  people: string;
  occurredAt: string | null;
  metadata: Record<string, unknown>;
  /** FTS5 bm25 score. Lower is a better match (bm25 returns negative values). */
  score: number;
  /** Highlighted excerpt from the body, for feeding an LLM as context. */
  snippet: string;
}

const DDL = `
CREATE TABLE IF NOT EXISTS doc (
  jmap_account_id TEXT NOT NULL,
  content_type    TEXT NOT NULL,
  id              TEXT NOT NULL,
  title           TEXT NOT NULL DEFAULT '',
  people          TEXT NOT NULL DEFAULT '',
  body            TEXT NOT NULL DEFAULT '',
  occurred_at     TEXT,
  metadata_json   TEXT NOT NULL DEFAULT '{}',
  indexed_at      INTEGER NOT NULL,
  PRIMARY KEY (jmap_account_id, content_type, id)
);
CREATE INDEX IF NOT EXISTS doc_recent
  ON doc(jmap_account_id, content_type, occurred_at DESC);

-- Standalone (not external-content) FTS5: the text is duplicated into this
-- table and kept in step manually on upsert. External content would avoid the
-- duplication but requires deleting the old FTS row using its OLD column
-- values, which an upsert does not have to hand - a well-known source of
-- silently-stale FTS rows. At this scale (a bounded recent window) the
-- duplication is the cheaper correctness trade.
CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5(
  title, people, body,
  tokenize='unicode61 remove_diacritics 2'
);

CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);
`;

export class MailIndexUnavailableError extends Error {
  constructor(message: string) {
    super(message);
    this.name = 'MailIndexUnavailableError';
  }
}

/**
 * Assert that the file we just opened is REALLY encrypted.
 *
 * This is not defensive boilerplate, it guards the sharpest landmine found
 * while designing this: on both `node:sqlite` and plain `better-sqlite3`,
 * `PRAGMA key = ...` is **silently accepted and does nothing** - no error, a
 * working database, and the mail sitting on disk in cleartext. Verified by
 * writing a file and recovering a canary string from the raw bytes.
 *
 * The check is on the VALUE, not the row count: a non-cipher binding returns
 * ZERO ROWS for `PRAGMA cipher_version`, so a naive `!== ''` comparison over a
 * missing row passes vacuously. Require a non-empty string.
 */
function assertEncrypted(db: SqlcipherDatabase, dbPath: string): void {
  const rows = db.pragma('cipher_version');
  const value =
    Array.isArray(rows) && rows.length > 0 && rows[0] && typeof rows[0] === 'object'
      ? (rows[0] as Record<string, unknown>).cipher_version
      : undefined;
  if (typeof value !== 'string' || value.trim().length === 0) {
    db.close();
    throw new MailIndexUnavailableError(
      `Refusing to use ${path.basename(dbPath)}: the SQLite binding reports no SQLCipher ` +
        `support (PRAGMA cipher_version returned ${JSON.stringify(rows)}), so the index ` +
        `would be written in cleartext.`,
    );
  }
}

export interface OpenOptions {
  storeDir: string;
  accountId: string;
  /** Raw 32-byte key. Used as SQLCipher's raw key (no KDF) via `PRAGMA key = "x'..'"`. */
  key: Buffer;
}

export class MailIndex {
  private constructor(
    private readonly db: SqlcipherDatabase,
    readonly dbPath: string,
  ) {}

  /**
   * Opens (creating if needed) the account's index. Throws
   * MailIndexUnavailableError when the native binding is absent or the file is
   * not actually encrypted; the caller turns the feature off rather than
   * falling back to something unencrypted.
   */
  static open({ storeDir, accountId, key }: OpenOptions): MailIndex {
    const Database = loadSqlcipher();
    if (!Database) {
      throw new MailIndexUnavailableError(
        '@signalapp/sqlcipher is not installed for this platform (it is an optional dependency).',
      );
    }
    if (key.length !== 32) {
      throw new MailIndexUnavailableError(`Index key must be 32 bytes, got ${key.length}.`);
    }

    const dbPath = indexDbPath(storeDir, accountId);
    fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: 0o700 });

    let db = new Database(dbPath);
    // The key pragma must be the FIRST statement on the connection. Hex form
    // means SQLCipher uses these 32 bytes as the raw key with no KDF, which is
    // right for a random key (a passphrase would want the KDF).
    db.pragma(`key = "x'${key.toString('hex')}'"`);
    assertEncrypted(db, dbPath);

    // A wrong key surfaces here rather than at open: SQLCipher only reads the
    // header lazily. Treat it as "unreadable" and rebuild from scratch - the
    // index is derived data, so there is nothing to recover and never anything
    // to prompt the user for (the key was never a user secret).
    let version: number | null;
    try {
      db.pragma('journal_mode = WAL');
      db.pragma('synchronous = NORMAL');
      // The offline replica (lib/offline-replica/**) is a SECOND connection to
      // this same file, writing disjoint tables. WAL lets a writer and readers
      // coexist, but two WRITERS get SQLITE_BUSY immediately without this - and
      // both subsystems are driven by the same renderer push handler, so they
      // genuinely do overlap.
      db.pragma('busy_timeout = 8000');
      version = readSchemaVersion(db);
    } catch {
      db.close();
      for (const f of dbSiblings(dbPath)) {
        try { fs.rmSync(f, { force: true }); } catch { /* best effort */ }
      }
      db = new Database(dbPath);
      db.pragma(`key = "x'${key.toString('hex')}'"`);
      assertEncrypted(db, dbPath);
      db.pragma('journal_mode = WAL');
      db.pragma('synchronous = NORMAL');
      // The offline replica (lib/offline-replica/**) is a SECOND connection to
      // this same file, writing disjoint tables. WAL lets a writer and readers
      // coexist, but two WRITERS get SQLITE_BUSY immediately without this - and
      // both subsystems are driven by the same renderer push handler, so they
      // genuinely do overlap.
      db.pragma('busy_timeout = 8000');
      version = null;
    }

    if (version !== null && version !== SCHEMA_VERSION) {
      // Rebuildable derived data: drop, don't migrate.
      db.exec('DROP TABLE IF EXISTS doc_fts; DROP TABLE IF EXISTS doc; DROP TABLE IF EXISTS meta;');
      version = null;
    }
    if (version === null) {
      db.exec(DDL);
      db.prepare('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)').run([
        'schema_version',
        String(SCHEMA_VERSION),
      ]);
    }

    return new MailIndex(db, dbPath);
  }

  close(): void {
    try { this.db.close(); } catch { /* already closed */ }
  }

  /**
   * Upserts documents and keeps the FTS rows in step. Returns the number of
   * rows written. One transaction for the whole batch - a partially-applied
   * batch is harmless (it is an index) but a transaction is faster.
   */
  upsert(docs: readonly IndexDoc[]): number {
    if (docs.length === 0) return 0;

    const upsertDoc = this.db.prepare(`
      INSERT INTO doc (jmap_account_id, content_type, id, title, people, body,
                       occurred_at, metadata_json, indexed_at)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
      ON CONFLICT(jmap_account_id, content_type, id) DO UPDATE SET
        title = excluded.title, people = excluded.people, body = excluded.body,
        occurred_at = excluded.occurred_at, metadata_json = excluded.metadata_json,
        indexed_at = excluded.indexed_at
      RETURNING rowid
    `);
    const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
    const insertFts = this.db.prepare(
      'INSERT INTO doc_fts (rowid, title, people, body) VALUES (?, ?, ?, ?)',
    );

    const now = Date.now();
    let written = 0;
    this.db.exec('BEGIN');
    try {
      for (const d of docs) {
        const row = upsertDoc.get([
          d.jmapAccountId, d.contentType, d.id,
          d.title, d.people, d.body,
          d.occurredAt, JSON.stringify(d.metadata ?? {}), now,
        ]);
        const rowid = row?.rowid;
        if (typeof rowid !== 'number') continue;
        // ON CONFLICT preserves the rowid, so delete-then-insert replaces the
        // old FTS row rather than accumulating duplicates for one document.
        deleteFts.run([rowid]);
        insertFts.run([rowid, d.title, d.people, d.body]);
        written++;
      }
      this.db.exec('COMMIT');
    } catch (error) {
      this.db.exec('ROLLBACK');
      throw error;
    }
    return written;
  }

  /** Removes documents by id (a JMAP `destroyed` id, or a stale row). */
  remove(jmapAccountId: string, contentType: ContentType, ids: readonly string[]): number {
    if (ids.length === 0) return 0;
    const findRow = this.db.prepare(
      'SELECT rowid FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
    );
    const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
    const deleteDoc = this.db.prepare(
      'DELETE FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
    );
    let removed = 0;
    this.db.exec('BEGIN');
    try {
      for (const id of ids) {
        const row = findRow.get([jmapAccountId, contentType, id]);
        if (typeof row?.rowid === 'number') deleteFts.run([row.rowid]);
        removed += deleteDoc.run([jmapAccountId, contentType, id]).changes;
      }
      this.db.exec('COMMIT');
    } catch (error) {
      this.db.exec('ROLLBACK');
      throw error;
    }
    return removed;
  }

  /**
   * Full-text search - the retrieval surface an AI feature calls to gather
   * context. `types` empty/omitted searches everything.
   */
  search(opts: {
    query: string;
    types?: readonly ContentType[];
    limit?: number;
    snippetTokens?: number;
  }): SearchHit[] {
    const match = toFtsMatchQuery(opts.query);
    if (!match) return [];

    const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200);
    const tokens = Math.min(Math.max(opts.snippetTokens ?? 24, 4), 64);
    const types = opts.types && opts.types.length > 0 ? opts.types : null;
    const typeFilter = types ? ` AND d.content_type IN (${types.map(() => '?').join(',')})` : '';

    // bm25 weights: a hit in the title or in a name/address is a stronger
    // signal than one in a long body, and for RAG the title is what makes a
    // retrieved chunk recognisable.
    const rows = this.db
      .prepare(`
        SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people,
               d.occurred_at, d.metadata_json,
               bm25(doc_fts, 8.0, 4.0, 1.0) AS score,
               snippet(doc_fts, 2, '[', ']', '…', ${tokens}) AS snip
        FROM doc_fts
        JOIN doc d ON d.rowid = doc_fts.rowid
        WHERE doc_fts MATCH ?${typeFilter}
        ORDER BY score ASC, d.occurred_at DESC
        LIMIT ?
      `)
      .all([match, ...(types ?? []), limit]);

    return rows.map((r) => ({
      contentType: String(r.content_type) as ContentType,
      id: String(r.id),
      jmapAccountId: String(r.jmap_account_id),
      title: String(r.title ?? ''),
      people: String(r.people ?? ''),
      occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at),
      metadata: safeParseObject(r.metadata_json),
      score: typeof r.score === 'number' ? r.score : 0,
      snippet: String(r.snip ?? ''),
    }));
  }

  /** Per-type counts and freshness, for the Settings UI and for debugging. */
  stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> {
    return this.db
      .prepare(`
        SELECT content_type, COUNT(*) AS n, MAX(occurred_at) AS newest, MAX(indexed_at) AS indexed
        FROM doc GROUP BY content_type ORDER BY content_type
      `)
      .all()
      .map((r) => ({
        contentType: String(r.content_type),
        count: Number(r.n ?? 0),
        newest: r.newest === null || r.newest === undefined ? null : String(r.newest),
        indexedAt: typeof r.indexed === 'number' ? r.indexed : null,
      }));
  }

  /** Ids already present, so a catch-up pass can skip re-fetching bodies. */
  existingIds(jmapAccountId: string, contentType: ContentType): Set<string> {
    const rows = this.db
      .prepare('SELECT id FROM doc WHERE jmap_account_id = ? AND content_type = ?')
      .all([jmapAccountId, contentType]);
    return new Set(rows.map((r) => String(r.id)));
  }

  /** Drops documents older than the retention floor for a type. */
  pruneOlderThan(jmapAccountId: string, contentType: ContentType, isoFloor: string): number {
    const rows = this.db
      .prepare(`
        SELECT rowid FROM doc
        WHERE jmap_account_id = ? AND content_type = ?
          AND occurred_at IS NOT NULL AND occurred_at < ?
      `)
      .all([jmapAccountId, contentType, isoFloor]);
    if (rows.length === 0) return 0;
    const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
    const deleteDoc = this.db.prepare('DELETE FROM doc WHERE rowid = ?');
    this.db.exec('BEGIN');
    try {
      for (const r of rows) {
        deleteFts.run([r.rowid]);
        deleteDoc.run([r.rowid]);
      }
      this.db.exec('COMMIT');
    } catch (error) {
      this.db.exec('ROLLBACK');
      throw error;
    }
    return rows.length;
  }
}

function readSchemaVersion(db: SqlcipherDatabase): number | null {
  try {
    const row = db.prepare("SELECT v FROM meta WHERE k = 'schema_version'").get();
    if (!row || row.v === undefined) return null;
    const n = Number(row.v);
    return Number.isFinite(n) ? n : null;
  } catch {
    // `meta` doesn't exist yet - a fresh file.
    return null;
  }
}

function safeParseObject(v: unknown): Record<string, unknown> {
  if (typeof v !== 'string') return {};
  try {
    const parsed = JSON.parse(v);
    return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
      ? (parsed as Record<string, unknown>)
      : {};
  } catch {
    return {};
  }
}

/**
 * Turns arbitrary user text into a safe FTS5 MATCH expression.
 *
 * FTS5's query syntax is not SQL, so parameter binding does NOT protect it: a
 * bare `"` or a stray `*`/`NEAR`/`:` in user input raises
 * `fts5: syntax error`, which would turn a normal search box into a 500. Every
 * token is quoted (making it a literal phrase) and a trailing `*` is added to
 * the last token so typing continues to match as the user types.
 *
 * Exported for unit testing - it is the one piece of this file with no
 * database dependency and the most ways to be wrong.
 */
export function toFtsMatchQuery(raw: string): string | null {
  if (typeof raw !== 'string') return null;
  // Split on anything that isn't a word character or an intra-word mark. Keeps
  // unicode letters (so "Müller" and "東京" survive) via the u flag.
  const tokens = raw
    .normalize('NFC')
    .split(/[^\p{L}\p{N}_@.'-]+/u)
    .map((t) => t.replace(/^['-]+|['-]+$/g, ''))
    .filter((t) => t.length > 0)
    .slice(0, 24);
  if (tokens.length === 0) return null;
  return tokens
    .map((t, i) => {
      const quoted = `"${t.replace(/"/g, '""')}"`;
      // Prefix-match only the final token, and only if it's long enough to not
      // match half the mailbox.
      return i === tokens.length - 1 && t.length >= 3 ? `${quoted}*` : quoted;
    })
    .join(' AND ');
}