SRCmail/lib/offline-replica/__tests__/store.test.ts

// Store-level invariants, against a REAL SQLCipher file.
//
// Skipped wholesale when the optional native binding is not installed (that is a
// normal state on a platform with no prebuild - see lib/mail-index/binding.ts), so
// this file must never be the only proof of anything.

import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { randomBytes } from 'node:crypto';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
import { indexDbPath } from '@/lib/mail-index/paths';
import { clampPolicy, DEFAULT_POLICY, ReplicaStore } from '../store';
import { reconcileStamp } from '../sync';
import { asChangesState, asSnapshotState, mintEnumerationCommitment } from '../states';
import type { EnvelopeRow } from '../types';

const ACCOUNT = 'alice@example.org';
const JMAP = 'jmap-account-1';

function envelope(id: string, receivedAt: string, extra: Partial<EnvelopeRow> = {}): EnvelopeRow {
  return {
    jmapAccountId: JMAP,
    id,
    threadId: `t-${id}`,
    receivedAt,
    size: 1000,
    subject: `subject ${id}`,
    preview: `preview ${id}`,
    fromJson: JSON.stringify([{ email: 'sender@example.org' }]),
    toJson: null,
    ccJson: null,
    blobId: `blob-${id}`,
    hasAttachment: false,
    keywordsJson: '{}',
    mailboxIds: ['inbox'],
    ...extra,
  };
}

describe.skipIf(!isSqlcipherAvailable())('ReplicaStore', () => {
  let storeDir: string;
  let key: Buffer;
  let store: ReplicaStore;

  beforeEach(() => {
    storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-test-'));
    key = randomBytes(32);
    store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
  });

  afterEach(() => {
    store.close();
    fs.rmSync(storeDir, { recursive: true, force: true });
  });

  it('writes into the SAME file as the search index, and it is really encrypted', () => {
    // One encryption boundary, one key, one purge. And `PRAGMA key` is a silent
    // no-op on a non-SQLCipher binding, so the header check is the only thing that
    // catches a store that "works" while sitting on disk in cleartext.
    expect(store.dbPath).toBe(indexDbPath(storeDir, ACCOUNT));
    store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
    store.close();
    const header = fs.readFileSync(store.dbPath).subarray(0, 15).toString('latin1');
    expect(header).not.toBe('SQLite format 3');
    const raw = Buffer.concat(
      ['', '-wal', '-shm']
        .map((s) => `${store.dbPath}${s}`)
        .filter((f) => fs.existsSync(f))
        .map((f) => fs.readFileSync(f)),
    );
    expect(raw.includes('subject e1')).toBe(false);
    // Re-open so afterEach's close() is harmless.
    store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
  });

  describe('cursor provenance at the storage layer', () => {
    it('refuses to create a cursor from nowhere', () => {
      // A cursor is born from seedCursor and nowhere else. Creating one in
      // advanceCursor would be a silent cursor-from-nowhere - exactly what the
      // branded types exist to make impossible.
      expect(() => store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s1')))
        .toThrow(/seed it first/);
    });

    it('writes the cursor AND the coverage row it justifies in one transaction', () => {
      store.transaction(() => {
        store.seedCursor(
          { jmapAccountId: JMAP, type: 'Email' },
          mintEnumerationCommitment({
            jmapAccountId: JMAP,
            snapshot: asSnapshotState('snap-1'),
            targetFrom: '2026-01-01T00:00:00.000Z',
            sweepFloor: '2026-01-01T00:00:00.000Z',
            kind: 'bootstrap',
          }),
          1000,
        );
      });
      expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('snap-1');
      const coverage = store.getCoverage(JMAP);
      expect(coverage?.phase).toBe('scanning');
      expect(coverage?.sweepFloor).toBe('2026-01-01T00:00:00.000Z');
    });

    it('rolls back a seed whose commitment is for the wrong account', () => {
      expect(() =>
        store.transaction(() => {
          store.seedCursor(
            { jmapAccountId: JMAP, type: 'Email' },
            mintEnumerationCommitment({
              jmapAccountId: 'someone-else',
              snapshot: asSnapshotState('snap'),
              targetFrom: 'x', sweepFloor: 'x', kind: 'bootstrap',
            }),
            1000,
          );
        }),
      ).toThrow(/different JMAP account/);
      expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull();
    });

    it('advances a seeded cursor and keeps counters field-level', () => {
      seed(store);
      store.transaction(() => {
        store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s2'));
        store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { consecutiveFailures: 3 });
      });
      const cursor = store.getCursor({ jmapAccountId: JMAP, type: 'Email' });
      expect(cursor?.state).toBe('s2');
      expect(cursor?.consecutiveFailures).toBe(3);
      // A patch must not be able to rewrite `state` - only advance/seed can.
      store.transaction(() => {
        store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { drainPending: true });
      });
      expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('s2');
    });
  });

  describe('envelope tier', () => {
    it('does NOT reset has_body on an idempotent replay', () => {
      // Otherwise a replayed page looks like "body missing" to the backfill job and
      // re-downloads every body in the page.
      store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
      store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{}}'); });
      expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10)).toHaveLength(0);
      store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 2); });
      expect(
        store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10),
        'a replayed envelope upsert must not clear has_body',
      ).toHaveLength(0);
    });

    it('patches only the two mutable properties, and no-ops for an absent id', () => {
      store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
      const ok = store.transaction(() =>
        store.patchEnvelopeMutable(JMAP, 'e1', { keywordsJson: '{"$seen":true}', mailboxIds: ['archive'] }),
      );
      expect(ok).toBe(true);
      expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual(['archive']);
      // An update for an id we do not hold must leave no membership rows behind.
      const missing = store.transaction(() =>
        store.patchEnvelopeMutable(JMAP, 'nope', { keywordsJson: '{}', mailboxIds: ['inbox'] }),
      );
      expect(missing).toBe(false);
      expect(store.mailboxIdsFor(JMAP, 'nope')).toEqual([]);
    });

    it('never writes a body whose envelope is gone', () => {
      // A body fetched moments before its envelope was destroyed in the same cycle
      // would otherwise land as an orphan.
      const wrote = store.transaction(() => store.putBodyIfEnvelopeExists(JMAP, 'ghost', '{}'));
      expect(wrote).toBe(false);
      expect(store.getBody(JMAP, 'ghost')).toBeNull();
    });

    it('deleting an email takes its body, membership and queue row with it', () => {
      store.transaction(() => {
        store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1);
        store.enqueueBodies([{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }]);
      });
      store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"a":1}'); });
      store.transaction(() => { store.deleteEmails(JMAP, ['e1']); });
      expect(store.getBody(JMAP, 'e1')).toBeNull();
      expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual([]);
      expect(store.countWantedBodies(JMAP, Date.now())).toBe(0);
    });
  });

  describe('the reconcile sweep', () => {
    it('refuses to run without a pinned stamp rather than deleting unverified records', () => {
      expect(() => store.sweep(JMAP, '2026-01-01T00:00:00.000Z', undefined))
        .toThrow(/refusing to delete unverified/);
    });

    it('keeps what the enumeration re-saw and deletes what it did not', () => {
      // The whole "seen set as one integer" trick: re-upserting refreshes
      // cached_at, and the sweep deletes anything still below the pin.
      store.transaction(() => {
        store.upsertEnvelopes([
          envelope('kept', '2026-08-01T00:00:00.000Z'),
          envelope('gone', '2026-08-02T00:00:00.000Z'),
        ], 100);
      });
      const stamp = Math.max(500, store.maxEnvelopeCachedAt(JMAP) + 1);
      store.transaction(() => {
        store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp);
      });
      store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); });
      expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull();
      expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull();
    });

    it('a stamp taken from a FROZEN clock would sweep nothing; the derived one works', () => {
      // Both halves matter, and both fail silently. Exercising the real
      // `reconcileStamp` rather than re-deriving it in the test is the point.
      store.transaction(() => {
        store.upsertEnvelopes([
          envelope('kept', '2026-08-01T00:00:00.000Z'),
          envelope('gone', '2026-08-02T00:00:00.000Z'),
        ], 9_999);
      });
      const frozenNow = 1_000;

      // The naive version: with the clock behind the data, nothing is below the
      // stamp, so a re-verified store sweeps zero rows and stale records live on.
      expect(store.sweep(JMAP, '2026-07-01T00:00:00.000Z', frozenNow)).toBe(0);

      const stamp = reconcileStamp(frozenNow, store.maxEnvelopeCachedAt(JMAP));
      expect(stamp).toBe(10_000);
      store.transaction(() => {
        store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp);
      });
      expect(store.transaction(() => store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp))).toBe(1);
      expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull();
      expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull();
    });

    it('stamping an enumeration with `now` instead of the pin deletes what it just verified', () => {
      // The other direction of the same bug: the pin EXCEEDS now, so a page that
      // stamps with `now` lands below the pin and the sweep eats it.
      store.transaction(() => {
        store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], 9_999);
      });
      const now = 1_000;
      const stamp = reconcileStamp(now, store.maxEnvelopeCachedAt(JMAP));
      // Re-verified against the server, but stamped with the WRONG value.
      store.transaction(() => {
        store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], now);
      });
      store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); });
      expect(
        store.getEnvelopeRaw(JMAP, 'verified'),
        'this is the failure mode the pinned stamp exists to prevent',
      ).toBeNull();
    });
  });

  describe('the body queue - the durable-terminal-state fixes', () => {
    it('enqueueBodies reports rows ACTUALLY INSERTED, not attempted', () => {
      // Reporting the attempted count made the mobile engine believe there was
      // unfinished work every cycle for as long as any envelope lacked a body,
      // chaining a new cycle every few seconds indefinitely.
      const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 };
      expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(1);
      expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(0);
    });

    it('never resets attempts on a re-enqueue', () => {
      const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 };
      store.transaction(() => { store.enqueueBodies([entry]); });
      store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 0, 'boom'); });
      store.transaction(() => { store.enqueueBodies([entry]); });
      expect(store.takeBodyQueue(JMAP, 10, Date.now())[0]?.attempts).toBe(1);
    });

    it('THE H1 REGRESSION: a gave-up row is KEPT and is never revived by a re-enqueue', () => {
      // Deleting the row on give-up was not enough: the backfill driver is
      // "envelope with no body", which cannot tell "not fetched yet" from
      // "deliberately not kept", so the next pass re-inserted a fresh attempts=0
      // row and a permanently-failing body was retried five times per cycle forever.
      store.transaction(() => {
        store.markBodyGaveUp(JMAP, [
          { emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' },
        ]);
      });
      expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['e1']);
      // Not WANTED any more, so the drain never picks it up again.
      expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0);
      // And a re-enqueue cannot resurrect it.
      const inserted = store.transaction(() =>
        store.enqueueBodies([
          { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
        ]),
      );
      expect(inserted).toBe(0);
      expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0);
    });

    it('THE H1c REGRESSION: a cap-shed body is markable even with NO existing queue row', () => {
      // The download/discard loop: the cap sheds a body that was fetched and stored
      // successfully, so there is no queue row left to UPDATE. If the mark is
      // silently dropped, the envelope is still inside the body WINDOW, the backfill
      // re-enqueues it, it downloads again, and the cap sheds it again - unbounded
      // data use that never terminates.
      store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
      store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{"1":{"value":"x"}}}'); });
      expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0); // no queue row exists

      store.transaction(() => {
        store.deleteBodies(JMAP, ['e1']);
        store.markBodyGaveUp(JMAP, [
          { emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' },
        ]);
      });
      expect(
        store.listBodyGiveUps(JMAP, 10),
        'the cap-shed mark must be an upsert, or the shed/re-download loop stays open',
      ).toEqual(['e1']);
      // The envelope is back to has_body=0 and still in the window, so without the
      // mark the backfill WOULD pick it up. With the mark it is excluded.
      expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10).map((e) => e.id))
        .toEqual(['e1']);
      expect(store.listBodyGiveUps(JMAP, 10)).toContain('e1');
    });

    it('clearing give-ups DELETES them, so they look like "never queued"', () => {
      // A cleared give-up must come back with a clean attempt count, which an
      // un-flag would not give.
      store.transaction(() => {
        store.markBodyGaveUp(JMAP, [
          { emailId: 'a', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' },
          { emailId: 'b', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' },
        ]);
      });
      store.transaction(() => { store.clearBodyGiveUps(JMAP, 'shed-by-cap'); });
      expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['a']);
      store.transaction(() => { store.clearBodyGiveUps(JMAP); });
      expect(store.listBodyGiveUps(JMAP, 10)).toEqual([]);
      expect(
        store.transaction(() =>
          store.enqueueBodies([
            { emailId: 'a', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
          ]),
        ),
        'a cleared give-up must be re-enqueueable',
      ).toBe(1);
    });

    it('honours a backoff window', () => {
      store.transaction(() => {
        store.enqueueBodies([
          { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
        ]);
      });
      store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 10_000, 'later'); });
      expect(store.takeBodyQueue(JMAP, 10, 5_000)).toHaveLength(0);
      expect(store.takeBodyQueue(JMAP, 10, 20_000)).toHaveLength(1);
    });
  });

  describe('eviction', () => {
    it('cap eviction takes the oldest bodies first and leaves envelopes alone', () => {
      store.transaction(() => {
        store.upsertEnvelopes([
          envelope('old', '2026-01-01T00:00:00.000Z'),
          envelope('new', '2026-08-01T00:00:00.000Z'),
        ], 1);
      });
      store.transaction(() => {
        store.putBodyIfEnvelopeExists(JMAP, 'old', '{"v":"old"}');
        store.putBodyIfEnvelopeExists(JMAP, 'new', '{"v":"new"}');
      });
      expect(store.oldestBodies(JMAP, 1).map((b) => b.emailId)).toEqual(['old']);
      store.transaction(() => { store.deleteBodies(JMAP, ['old']); });
      // The message stays LISTED - only its content went.
      expect(store.getEnvelopeRaw(JMAP, 'old')).not.toBeNull();
      expect(store.countBodies(JMAP)).toBe(1);
    });

    it('no deletion path leaves an orphan body behind', () => {
      // This is the real invariant. `orphanBodies()` is a belt-and-braces sweep for
      // orphans a CRASH between two transactions could leave; it is deliberately
      // not reachable through the store's own API, which is what this asserts.
      // (So the detection query itself is covered only by the integration run, not
      // by this file - stated rather than papered over with a vacuous assertion.)
      store.transaction(() => {
        store.upsertEnvelopes([
          envelope('a', '2026-01-01T00:00:00.000Z'),
          envelope('b', '2026-08-01T00:00:00.000Z'),
        ], 1);
      });
      store.transaction(() => {
        store.putBodyIfEnvelopeExists(JMAP, 'a', '{"v":1}');
        store.putBodyIfEnvelopeExists(JMAP, 'b', '{"v":2}');
      });
      expect(store.countBodies(JMAP)).toBe(2);

      store.transaction(() => { store.deleteEmails(JMAP, ['a']); });
      expect(store.orphanBodies(JMAP, 10)).toEqual([]);

      store.transaction(() => { store.evictEnvelopesBelow(JMAP, '2026-09-01T00:00:00.000Z'); });
      expect(store.countEnvelopes(JMAP)).toBe(0);
      expect(store.countBodies(JMAP)).toBe(0);
      expect(store.orphanBodies(JMAP, 10)).toEqual([]);
    });
  });

  describe('purge', () => {
    it('purgeAll takes the CURSORS with the records', () => {
      // A record wipe that leaves a live cursor behind is the one state no amount
      // of syncing repairs: /changes cannot re-deliver mail that already existed
      // when the cursor was captured.
      seed(store);
      store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
      store.transaction(() => { store.purgeAll(); });
      expect(store.countEnvelopes(JMAP)).toBe(0);
      expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull();
      expect(store.getCoverage(JMAP)).toBeNull();
    });

    it('a wrong key is treated as unreadable and rebuilt, never as a prompt', () => {
      store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
      store.close();
      const other = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key: randomBytes(32) });
      try {
        expect(other.countEnvelopes(JMAP)).toBe(0);
      } finally {
        other.close();
      }
      store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
    });
  });

  describe('policy', () => {
    it('round-trips and clamps', () => {
      store.transaction(() => { store.setPolicy({ envelopeDays: 99999, bodyDays: 0, maxBodyMB: 1 }); });
      const policy = store.getPolicy();
      expect(policy.envelopeDays).toBe(3650);
      expect(policy.bodyDays).toBe(1);
      expect(policy.maxBodyMB).toBe(16);
    });

    it('defaults when nothing was ever written', () => {
      expect(store.getPolicy()).toEqual(DEFAULT_POLICY);
    });
  });

  describe('read path', () => {
    it('lists a mailbox page newest-first with a correct total', () => {
      store.transaction(() => {
        store.upsertEnvelopes([
          envelope('a', '2026-08-01T00:00:00.000Z'),
          envelope('b', '2026-08-02T00:00:00.000Z'),
          envelope('c', '2026-08-03T00:00:00.000Z', { mailboxIds: ['archive'] }),
        ], 1);
      });
      const inbox = store.listEnvelopes(JMAP, 'inbox', 10, 0);
      expect(inbox.total).toBe(2);
      expect(inbox.rows.map((r) => String(r.id))).toEqual(['b', 'a']);
      // A null mailbox is "everything", which is what the unified views want.
      expect(store.listEnvelopes(JMAP, null, 10, 0).total).toBe(3);
      expect(store.listEnvelopes(JMAP, 'archive', 10, 0).rows.map((r) => String(r.id))).toEqual(['c']);
    });

    it('reports the account ids it holds without needing a network session', () => {
      store.transaction(() => { store.upsertEnvelopes([envelope('a', '2026-08-01T00:00:00.000Z')], 1); });
      expect(store.knownJmapAccountIds()).toEqual([JMAP]);
    });
  });
});

describe('clampPolicy', () => {
  it('never lets the body window exceed the envelope window', () => {
    expect(clampPolicy({ envelopeDays: 30, bodyDays: 365, maxBodyMB: 100 }).bodyDays).toBe(30);
  });

  it('falls back to defaults for junk input', () => {
    expect(clampPolicy({ envelopeDays: NaN } as never).envelopeDays).toBe(DEFAULT_POLICY.envelopeDays);
    expect(clampPolicy(null)).toEqual(DEFAULT_POLICY);
    expect(clampPolicy(undefined)).toEqual(DEFAULT_POLICY);
  });
});

function seed(store: ReplicaStore): void {
  store.transaction(() => {
    for (const type of ['Email', 'Mailbox'] as const) {
      store.seedCursor(
        { jmapAccountId: JMAP, type },
        mintEnumerationCommitment({
          jmapAccountId: JMAP,
          snapshot: asSnapshotState('snap'),
          targetFrom: '2026-01-01T00:00:00.000Z',
          sweepFloor: '2026-01-01T00:00:00.000Z',
          kind: 'bootstrap',
        }),
        1000,
      );
    }
  });
}