SRCmail/lib/offline-replica/schema.ts

// The offline mail replica's schema.
//
// WHY IT LIVES IN THE SAME ENCRYPTED FILE AS THE SEARCH INDEX
// (`lib/mail-index/paths.ts`'s `indexDbPath`), on its own connection:
//
//  * One encryption boundary, one key, one keychain entry, one purge. A second
//    keystore would double the number of places a key can be mishandled for no
//    gain - and `electron/key-service.ts` + the fd-3 channel already work.
//  * `sync_state` (cursors, coverage, flags) sits in the SAME FILE as the
//    records it describes. That is load-bearing, not tidiness: deleting the file
//    removes cursors and records together, so a cursor can never survive a wipe
//    and then be advanced over changes that will never be re-delivered. A
//    cursor in a sidecar JSON file is exactly the class of bug the mobile
//    client's S1 finding is about.
//  * The tables are DISJOINT from the index's (`doc`, `doc_fts`), so the two
//    subsystems never contend for a row - only, briefly, for SQLite's write
//    lock, which `PRAGMA busy_timeout` resolves. Both open with WAL.
//
// The one coupling to accept: `lib/mail-index/store.ts` drops `meta` on an index
// schema bump, which takes `replica_schema_version` with it. The replica reads
// that as "version missing" and purges + re-bootstraps - correct, just wasteful,
// and only on an index schema change. What must NOT happen is records surviving
// while the version row vanishes, which is why the purge below is all-or-nothing
// and includes `replica_sync_state`.
//
// Deliberately NO foreign keys from `replica_email_mailbox.mailbox_id` ->
// `replica_mailbox`, and no cascade from `replica_envelope`. The two change
// streams are not transactionally coupled, so a membership row referencing a
// not-yet-fetched or already-destroyed mailbox is a NORMAL transient state; an
// FK would turn correct behaviour into a constraint violation, and a cascade on
// mailbox deletion would delete mail, violating the deletion-provenance rule
// (only the Email stream may delete an email).

/** Bumped on any incompatible change. Mismatch = purge + re-bootstrap, never migrate. */
export const REPLICA_SCHEMA_VERSION = 1;

export const REPLICA_VERSION_KEY = 'replica_schema_version';

export const REPLICA_DDL = `
CREATE TABLE IF NOT EXISTS replica_mailbox (
  jmap_account_id TEXT NOT NULL,
  id              TEXT NOT NULL,
  name            TEXT NOT NULL DEFAULT '',
  parent_id       TEXT,
  role            TEXT,
  sort_order      INTEGER,
  total_emails    INTEGER,
  unread_emails   INTEGER,
  total_threads   INTEGER,
  unread_threads  INTEGER,
  my_rights_json  TEXT,
  is_subscribed   INTEGER,
  PRIMARY KEY (jmap_account_id, id)
);

-- The envelope tier: everything EMAIL_LIST_PROPERTIES carries, so an offline
-- message list renders exactly as an online one does.
CREATE TABLE IF NOT EXISTS replica_envelope (
  jmap_account_id TEXT NOT NULL,
  id              TEXT NOT NULL,
  thread_id       TEXT,
  received_at     TEXT NOT NULL,
  size            INTEGER,
  subject         TEXT,
  preview         TEXT,
  from_json       TEXT,
  to_json         TEXT,
  cc_json         TEXT,
  blob_id         TEXT,
  has_attachment  INTEGER NOT NULL DEFAULT 0,
  keywords_json   TEXT NOT NULL DEFAULT '{}',
  -- Owned by the BODY tier. Excluded from the envelope upsert's DO UPDATE SET,
  -- or an idempotent page replay would look like "body missing" to the backfill
  -- job and re-download every body in the page.
  has_body        INTEGER NOT NULL DEFAULT 0,
  body_bytes      INTEGER NOT NULL DEFAULT 0,
  cached_at       INTEGER NOT NULL,
  PRIMARY KEY (jmap_account_id, id)
);
CREATE INDEX IF NOT EXISTS replica_envelope_received
  ON replica_envelope(jmap_account_id, received_at DESC);
-- The body-backfill driver: envelopes inside the body window with no body yet.
CREATE INDEX IF NOT EXISTS replica_envelope_nobody
  ON replica_envelope(jmap_account_id, has_body, received_at DESC);

-- Membership is its own table: an email is in many mailboxes, and listing by
-- folder must be an index seek rather than a scan of every cached row.
CREATE TABLE IF NOT EXISTS replica_email_mailbox (
  jmap_account_id TEXT NOT NULL,
  email_id        TEXT NOT NULL,
  mailbox_id      TEXT NOT NULL,
  PRIMARY KEY (jmap_account_id, email_id, mailbox_id)
);
CREATE INDEX IF NOT EXISTS replica_email_mailbox_by_mailbox
  ON replica_email_mailbox(jmap_account_id, mailbox_id);

-- The body tier. "received_at" is duplicated here on purpose so cap eviction is
-- a single-table ordered scan that cannot be blinded by a missing join.
CREATE TABLE IF NOT EXISTS replica_body (
  jmap_account_id TEXT NOT NULL,
  email_id        TEXT NOT NULL,
  received_at     TEXT NOT NULL,
  json            TEXT NOT NULL,
  bytes           INTEGER NOT NULL,
  PRIMARY KEY (jmap_account_id, email_id)
);
CREATE INDEX IF NOT EXISTS replica_body_received
  ON replica_body(jmap_account_id, received_at ASC);

-- "gave_up" is what makes a body-tier TERMINAL STATE durable, and it is the fix
-- for the worst bug found on the mobile client. Deleting the queue row on
-- give-up was not enough: the backfill job's driver is "envelope with no body",
-- a predicate that CANNOT distinguish "not fetched yet" from "deliberately not
-- kept". So the next pass re-inserted a fresh attempts=0 row and a
-- permanently-failing body was retried five times per cycle, forever. Same
-- shape for a "notFound" body, and worst of all for a body shed by the size cap:
-- shed -> still inside the body window -> re-enqueued -> re-downloaded -> shed
-- again. Unbounded data use with no termination. Keeping the row with a flag is
-- what closes all three.
CREATE TABLE IF NOT EXISTS replica_body_queue (
  jmap_account_id TEXT NOT NULL,
  email_id        TEXT NOT NULL,
  received_at     TEXT NOT NULL,
  attempts        INTEGER NOT NULL DEFAULT 0,
  next_attempt_at INTEGER,
  last_error      TEXT,
  gave_up         INTEGER NOT NULL DEFAULT 0,
  gave_up_reason  TEXT,
  PRIMARY KEY (jmap_account_id, email_id)
);
CREATE INDEX IF NOT EXISTS replica_body_queue_wanted
  ON replica_body_queue(jmap_account_id, gave_up, received_at DESC);

-- Cursors, coverage and flags. Row-per-field, NOT one JSON blob: a blob loaded
-- at cycle start and written at cycle end silently reverts any concurrent write
-- to a different field. On the mobile client that produced an empty record store
-- with a live advanced cursor and resyncRequired reset to false - a permanent
-- silent data gap that is unreachable by design.
CREATE TABLE IF NOT EXISTS replica_sync_state (
  k TEXT PRIMARY KEY,
  v TEXT NOT NULL
);
`;

/**
 * Everything a purge removes. `replica_sync_state` is INCLUDED: a record wipe
 * that leaves cursors behind is the one state from which no amount of syncing
 * recovers, because `/changes` structurally cannot re-deliver mail that already
 * existed when the cursor was captured.
 */
export const REPLICA_TABLES: readonly string[] = [
  'replica_envelope',
  'replica_email_mailbox',
  'replica_body',
  'replica_body_queue',
  'replica_mailbox',
  'replica_sync_state',
];

/** Record tables only - used when a reconcile rebuilds without discarding policy. */
export const REPLICA_RECORD_TABLES: readonly string[] = [
  'replica_envelope',
  'replica_email_mailbox',
  'replica_body',
  'replica_body_queue',
  'replica_mailbox',
];

export const FLAGS_KEY = 'flags';
export const POLICY_KEY = 'policy';

export function cursorStateKey(jmapAccountId: string, type: string): string {
  return `cursor:${jmapAccountId}:${type}`;
}

export function coverageStateKey(jmapAccountId: string): string {
  return `coverage:${jmapAccountId}`;
}