// The offline mail replica's schema. // // WHY IT LIVES IN THE SAME ENCRYPTED FILE AS THE SEARCH INDEX // (`lib/mail-index/paths.ts`'s `indexDbPath`), on its own connection: // // * One encryption boundary, one key, one keychain entry, one purge. A second // keystore would double the number of places a key can be mishandled for no // gain - and `electron/key-service.ts` + the fd-3 channel already work. // * `sync_state` (cursors, coverage, flags) sits in the SAME FILE as the // records it describes. That is load-bearing, not tidiness: deleting the file // removes cursors and records together, so a cursor can never survive a wipe // and then be advanced over changes that will never be re-delivered. A // cursor in a sidecar JSON file is exactly the class of bug the mobile // client's S1 finding is about. // * The tables are DISJOINT from the index's (`doc`, `doc_fts`), so the two // subsystems never contend for a row - only, briefly, for SQLite's write // lock, which `PRAGMA busy_timeout` resolves. Both open with WAL. // // The one coupling to accept: `lib/mail-index/store.ts` drops `meta` on an index // schema bump, which takes `replica_schema_version` with it. The replica reads // that as "version missing" and purges + re-bootstraps - correct, just wasteful, // and only on an index schema change. What must NOT happen is records surviving // while the version row vanishes, which is why the purge below is all-or-nothing // and includes `replica_sync_state`. // // Deliberately NO foreign keys from `replica_email_mailbox.mailbox_id` -> // `replica_mailbox`, and no cascade from `replica_envelope`. The two change // streams are not transactionally coupled, so a membership row referencing a // not-yet-fetched or already-destroyed mailbox is a NORMAL transient state; an // FK would turn correct behaviour into a constraint violation, and a cascade on // mailbox deletion would delete mail, violating the deletion-provenance rule // (only the Email stream may delete an email). /** Bumped on any incompatible change. Mismatch = purge + re-bootstrap, never migrate. */ export const REPLICA_SCHEMA_VERSION = 1; export const REPLICA_VERSION_KEY = 'replica_schema_version'; export const REPLICA_DDL = ` CREATE TABLE IF NOT EXISTS replica_mailbox ( jmap_account_id TEXT NOT NULL, id TEXT NOT NULL, name TEXT NOT NULL DEFAULT '', parent_id TEXT, role TEXT, sort_order INTEGER, total_emails INTEGER, unread_emails INTEGER, total_threads INTEGER, unread_threads INTEGER, my_rights_json TEXT, is_subscribed INTEGER, PRIMARY KEY (jmap_account_id, id) ); -- The envelope tier: everything EMAIL_LIST_PROPERTIES carries, so an offline -- message list renders exactly as an online one does. CREATE TABLE IF NOT EXISTS replica_envelope ( jmap_account_id TEXT NOT NULL, id TEXT NOT NULL, thread_id TEXT, received_at TEXT NOT NULL, size INTEGER, subject TEXT, preview TEXT, from_json TEXT, to_json TEXT, cc_json TEXT, blob_id TEXT, has_attachment INTEGER NOT NULL DEFAULT 0, keywords_json TEXT NOT NULL DEFAULT '{}', -- Owned by the BODY tier. Excluded from the envelope upsert's DO UPDATE SET, -- or an idempotent page replay would look like "body missing" to the backfill -- job and re-download every body in the page. has_body INTEGER NOT NULL DEFAULT 0, body_bytes INTEGER NOT NULL DEFAULT 0, cached_at INTEGER NOT NULL, PRIMARY KEY (jmap_account_id, id) ); CREATE INDEX IF NOT EXISTS replica_envelope_received ON replica_envelope(jmap_account_id, received_at DESC); -- The body-backfill driver: envelopes inside the body window with no body yet. CREATE INDEX IF NOT EXISTS replica_envelope_nobody ON replica_envelope(jmap_account_id, has_body, received_at DESC); -- Membership is its own table: an email is in many mailboxes, and listing by -- folder must be an index seek rather than a scan of every cached row. CREATE TABLE IF NOT EXISTS replica_email_mailbox ( jmap_account_id TEXT NOT NULL, email_id TEXT NOT NULL, mailbox_id TEXT NOT NULL, PRIMARY KEY (jmap_account_id, email_id, mailbox_id) ); CREATE INDEX IF NOT EXISTS replica_email_mailbox_by_mailbox ON replica_email_mailbox(jmap_account_id, mailbox_id); -- The body tier. "received_at" is duplicated here on purpose so cap eviction is -- a single-table ordered scan that cannot be blinded by a missing join. CREATE TABLE IF NOT EXISTS replica_body ( jmap_account_id TEXT NOT NULL, email_id TEXT NOT NULL, received_at TEXT NOT NULL, json TEXT NOT NULL, bytes INTEGER NOT NULL, PRIMARY KEY (jmap_account_id, email_id) ); CREATE INDEX IF NOT EXISTS replica_body_received ON replica_body(jmap_account_id, received_at ASC); -- "gave_up" is what makes a body-tier TERMINAL STATE durable, and it is the fix -- for the worst bug found on the mobile client. Deleting the queue row on -- give-up was not enough: the backfill job's driver is "envelope with no body", -- a predicate that CANNOT distinguish "not fetched yet" from "deliberately not -- kept". So the next pass re-inserted a fresh attempts=0 row and a -- permanently-failing body was retried five times per cycle, forever. Same -- shape for a "notFound" body, and worst of all for a body shed by the size cap: -- shed -> still inside the body window -> re-enqueued -> re-downloaded -> shed -- again. Unbounded data use with no termination. Keeping the row with a flag is -- what closes all three. CREATE TABLE IF NOT EXISTS replica_body_queue ( jmap_account_id TEXT NOT NULL, email_id TEXT NOT NULL, received_at TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, next_attempt_at INTEGER, last_error TEXT, gave_up INTEGER NOT NULL DEFAULT 0, gave_up_reason TEXT, PRIMARY KEY (jmap_account_id, email_id) ); CREATE INDEX IF NOT EXISTS replica_body_queue_wanted ON replica_body_queue(jmap_account_id, gave_up, received_at DESC); -- Cursors, coverage and flags. Row-per-field, NOT one JSON blob: a blob loaded -- at cycle start and written at cycle end silently reverts any concurrent write -- to a different field. On the mobile client that produced an empty record store -- with a live advanced cursor and resyncRequired reset to false - a permanent -- silent data gap that is unreachable by design. CREATE TABLE IF NOT EXISTS replica_sync_state ( k TEXT PRIMARY KEY, v TEXT NOT NULL ); `; /** * Everything a purge removes. `replica_sync_state` is INCLUDED: a record wipe * that leaves cursors behind is the one state from which no amount of syncing * recovers, because `/changes` structurally cannot re-deliver mail that already * existed when the cursor was captured. */ export const REPLICA_TABLES: readonly string[] = [ 'replica_envelope', 'replica_email_mailbox', 'replica_body', 'replica_body_queue', 'replica_mailbox', 'replica_sync_state', ]; /** Record tables only - used when a reconcile rebuilds without discarding policy. */ export const REPLICA_RECORD_TABLES: readonly string[] = [ 'replica_envelope', 'replica_email_mailbox', 'replica_body', 'replica_body_queue', 'replica_mailbox', ]; export const FLAGS_KEY = 'flags'; export const POLICY_KEY = 'policy'; export function cursorStateKey(jmapAccountId: string, type: string): string { return `cursor:${jmapAccountId}:${type}`; } export function coverageStateKey(jmapAccountId: string): string { return `coverage:${jmapAccountId}`; }