diff --git a/app/(main)/[locale]/page.tsx b/app/(main)/[locale]/page.tsx index 3d8250b9..74843917 100644 --- a/app/(main)/[locale]/page.tsx +++ b/app/(main)/[locale]/page.tsx @@ -1080,6 +1080,21 @@ export default function Home() { } catch { /* the index is optional */ } + // The offline REPLICA's launch catch-up. Same reasoning as the index's, + // plus one of its own: a `/changes` cursor cannot tell us about anything + // that happened while the process was dead, so a cycle at launch is what + // drains the backlog. One cycle is bounded, so a first sync of a large + // mailbox needs several - `chainSync` runs them with a hard cap. + // + // Sequenced AFTER the index rather than in parallel: both write the same + // SQLite file, and although `busy_timeout` makes concurrent writers safe, + // there is no reason to spend the contention during first paint. + try { + const { chainSync } = await import('@/lib/offline-replica-client'); + await chainSync({ slot: useAccountStore.getState().getActiveAccount()?.cookieSlot }); + } catch { + /* the replica is optional */ + } })(); // Deliberately after the initial mailbox fetch settles: the catch-up is a // background nicety and must not compete with first paint. diff --git a/app/api/offline/mail/route.ts b/app/api/offline/mail/route.ts new file mode 100644 index 00000000..b5bc1124 --- /dev/null +++ b/app/api/offline/mail/route.ts @@ -0,0 +1,87 @@ +// GET /api/offline/mail?kind=mailboxes|list|message - the OFFLINE READ SURFACE. +// +// THIS ROUTE MUST NEVER MAKE A NETWORK CALL. That is the whole feature: it is +// consulted precisely when the backend is unreachable, so a JMAP session fetch to +// learn the account id would fail for the exact reason the route was called. The +// account is resolved from the request's own encrypted `jmap_stalwart_ctx` cookie +// (a local decrypt) and from the account ids the store already holds rows for. +// +// It is a FALLBACK, not a cache in front of the server - see `read.ts`'s header +// for the coherence rules that depend on that, and `lib/offline-fallback-client.ts` +// for the one place that decides a read has genuinely failed. +import { NextRequest, NextResponse } from 'next/server'; +import { + readEnvelopePage, readMailboxes, readMessage, +} from '@/lib/offline-replica/read'; +import { + resolveIndexSession, resolveReadAccountId, withReplica, +} from '@/lib/offline-replica/engine'; +import { gateReplicaRoute, NO_STORE, replicaErrorResponse } from '@/lib/offline-replica/route-gate'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +const MAX_LIMIT = 200; + +export async function GET(request: NextRequest) { + const gated = gateReplicaRoute(); + if (gated) return gated; + + const params = request.nextUrl.searchParams; + const kind = params.get('kind') ?? 'mailboxes'; + if (kind !== 'mailboxes' && kind !== 'list' && kind !== 'message') { + return NextResponse.json({ error: 'kind must be mailboxes, list or message' }, { status: 400 }); + } + + try { + const session = await resolveIndexSession(request); + const payload = await withReplica(session.accountId, (store) => { + const jmapAccountId = resolveReadAccountId(store, params.get('jmapAccountId')); + if (!jmapAccountId) { + // Nothing synced yet for this account. Not an error - the caller falls + // back to whatever it would have shown without a replica. + return { empty: true as const }; + } + + if (kind === 'mailboxes') { + return { empty: false as const, jmapAccountId, mailboxes: readMailboxes(store, jmapAccountId) }; + } + + if (kind === 'list') { + const rawLimit = Number(params.get('limit') ?? '50'); + const limit = Number.isFinite(rawLimit) + ? Math.min(Math.max(Math.trunc(rawLimit), 1), MAX_LIMIT) + : 50; + const rawOffset = Number(params.get('offset') ?? '0'); + const offset = Number.isFinite(rawOffset) ? Math.max(Math.trunc(rawOffset), 0) : 0; + // An absent mailboxId means "everything", which is what the unified views + // ask for; an empty string is a caller bug and must not silently widen. + const mailboxParam = params.get('mailboxId'); + const mailboxId = mailboxParam === null ? null : mailboxParam; + if (mailboxId === '') { + return { empty: true as const }; + } + const page = readEnvelopePage(store, jmapAccountId, mailboxId, limit, offset); + return { empty: false as const, jmapAccountId, ...page }; + } + + const id = params.get('id'); + if (!id || id.length > 256) return { empty: true as const }; + const message = readMessage(store, jmapAccountId, id); + if (!message) return { empty: false as const, jmapAccountId, email: null, hasBody: false }; + return { + empty: false as const, + jmapAccountId, + email: message.email, + hasBody: message.hasBody, + }; + }); + + if (payload.empty) { + return NextResponse.json({ ok: true, available: false }, { headers: NO_STORE }); + } + return NextResponse.json({ ok: true, available: true, ...payload }, { headers: NO_STORE }); + } catch (error) { + return replicaErrorResponse(error, 'offline read'); + } +} diff --git a/app/api/offline/status/route.ts b/app/api/offline/status/route.ts new file mode 100644 index 00000000..7c493e43 --- /dev/null +++ b/app/api/offline/status/route.ts @@ -0,0 +1,104 @@ +// GET /api/offline/status - size, freshness and retention policy, for Settings. +// PUT /api/offline/status - update the retention policy. +// DELETE /api/offline/status - purge the replica. +// +// The POLICY LIVES IN THE ENCRYPTED STORE, not in renderer localStorage. The +// design review's H1 was that a server-side engine cannot read a renderer-only +// setting; keeping the policy server-side means the retention pass always has the +// value it needs, while the DECISION TO SYNC AT ALL stays with the renderer, so +// nothing is ever materialised for an account that never opted in. +// +// Like every read here, GET makes no network call: an offline user must still be +// able to see what they have and free the space. +import { NextRequest, NextResponse } from 'next/server'; +import { clampPolicy, POLICY_LIMITS, type RetentionPolicy } from '@/lib/offline-replica/store'; +import { resolveIndexSession, resolveReadAccountId, withReplica } from '@/lib/offline-replica/engine'; +import { gateReplicaRoute, NO_STORE, replicaErrorResponse } from '@/lib/offline-replica/route-gate'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +export async function GET(request: NextRequest) { + const gated = gateReplicaRoute(); + if (gated) return gated; + try { + const session = await resolveIndexSession(request); + const payload = await withReplica(session.accountId, (store) => { + const jmapAccountId = resolveReadAccountId(store, null); + const policy = store.getPolicy(); + const flags = store.getFlags(Date.now()); + if (!jmapAccountId) { + return { + policy, + limits: POLICY_LIMITS, + synced: false, + stats: null, + coveragePhase: 'never-run', + resyncRequired: flags.resyncRequired, + lastCycleAt: flags.lastCycleAt ?? null, + lastCycleOk: flags.lastCycleOk ?? null, + }; + } + return { + policy, + limits: POLICY_LIMITS, + synced: true, + stats: store.stats(jmapAccountId), + coveragePhase: store.getCoverage(jmapAccountId)?.phase ?? 'never-run', + coveredFrom: store.getCoverage(jmapAccountId)?.coveredFrom ?? null, + resyncRequired: flags.resyncRequired, + lastCycleAt: flags.lastCycleAt ?? null, + lastCycleOk: flags.lastCycleOk ?? null, + lastCycleError: flags.lastCycleError ?? null, + }; + }); + return NextResponse.json({ ok: true, ...payload }, { headers: NO_STORE }); + } catch (error) { + return replicaErrorResponse(error, 'offline status'); + } +} + +export async function PUT(request: NextRequest) { + const gated = gateReplicaRoute(); + if (gated) return gated; + + let body: Record = {}; + try { + const text = await request.text(); + if (text.trim()) body = JSON.parse(text) as Record; + } catch { + return NextResponse.json({ error: 'Malformed JSON body' }, { status: 400 }); + } + + const policy = clampPolicy(body as Partial); + try { + const session = await resolveIndexSession(request); + await withReplica(session.accountId, (store) => { + store.transaction(() => { store.setPolicy(policy); }); + }); + // The cycle applies it: a widen re-enters coverage scanning, a narrow evicts, + // and the clock guard is told this was INTENT rather than a glitch by the + // `lastEnvelopeDays` it compares against. + return NextResponse.json({ ok: true, policy }, { headers: NO_STORE }); + } catch (error) { + return replicaErrorResponse(error, 'offline policy update'); + } +} + +export async function DELETE(request: NextRequest) { + const gated = gateReplicaRoute(); + if (gated) return gated; + try { + const session = await resolveIndexSession(request); + await withReplica(session.accountId, (store) => { + // ALL OF IT, cursors included. A record wipe that leaves cursors behind is + // the one state no amount of syncing repairs: `/changes` structurally cannot + // re-deliver mail that already existed when the cursor was captured, so the + // next cycle would advance a live cursor over an empty store forever. + store.transaction(() => { store.purgeAll(); }); + }); + return NextResponse.json({ ok: true, purged: true }, { headers: NO_STORE }); + } catch (error) { + return replicaErrorResponse(error, 'offline purge'); + } +} diff --git a/app/api/offline/sync/route.ts b/app/api/offline/sync/route.ts new file mode 100644 index 00000000..de895463 --- /dev/null +++ b/app/api/offline/sync/route.ts @@ -0,0 +1,49 @@ +// POST /api/offline/sync - run ONE bounded delta-sync cycle for the calling +// session's account. +// +// The renderer drives this: once at launch (catch-up for whatever changed while +// the app was closed, for which no push event was ever delivered) and on each +// JMAP `StateChange` from the live push connection. There is no background worker +// and no resident credential - see `lib/offline-replica/sync.ts`'s header for why +// that architecture choice keeps most of the original design review's critical +// findings out of scope entirely. +// +// A cycle is BOUNDED. `unfinishedWork: true` means "call again", and the renderer +// chains with a cap; it never means an error. +import { NextRequest, NextResponse } from 'next/server'; +import { clampPolicy, type RetentionPolicy } from '@/lib/offline-replica/store'; +import { resolveIndexSession, syncAccount } from '@/lib/offline-replica/engine'; +import { gateReplicaRoute, NO_STORE, replicaErrorResponse } from '@/lib/offline-replica/route-gate'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +export async function POST(request: NextRequest) { + const gated = gateReplicaRoute(); + if (gated) return gated; + + let body: Record = {}; + try { + const text = await request.text(); + if (text.trim()) body = JSON.parse(text) as Record; + } catch { + return NextResponse.json({ error: 'Malformed JSON body' }, { status: 400 }); + } + + const rawPolicy = body.policy; + const policy: RetentionPolicy | undefined = + rawPolicy && typeof rawPolicy === 'object' && !Array.isArray(rawPolicy) + ? clampPolicy(rawPolicy as Partial) + : undefined; + + try { + const session = await resolveIndexSession(request); + const report = await syncAccount(session, { + policy, + forceResync: body.forceResync === true, + }); + return NextResponse.json({ ok: report.ok, report }, { headers: NO_STORE }); + } catch (error) { + return replicaErrorResponse(error, 'sync'); + } +} diff --git a/components/settings/local-index-settings.tsx b/components/settings/local-index-settings.tsx index 670dbeac..59e529ce 100644 --- a/components/settings/local-index-settings.tsx +++ b/components/settings/local-index-settings.tsx @@ -15,6 +15,10 @@ import { SettingsSection, SettingItem } from './settings-section'; import { isElectronShell } from '@/lib/electron-bridge'; import { useAccountStore } from '@/stores/account-store'; import { catchUpIndex, fetchIndexStats, type IndexStats } from '@/lib/mail-index-client'; +import { + chainSync, fetchReplicaStatus, purgeReplica, updateRetentionPolicy, + type ReplicaStatus, type RetentionPolicy, +} from '@/lib/offline-replica-client'; const TYPE_LABELS: Record = { mail: 'Mail', @@ -118,6 +122,190 @@ export function LocalIndexSettings() { {busy ? 'Indexing…' : 'Update index'} + + ); } + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + const units = ['KB', 'MB', 'GB']; + let value = bytes / 1024; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit++; } + return `${value.toFixed(value >= 10 ? 0 : 1)} ${units[unit]}`; +} + +const PHASE_LABELS: Record = { + 'never-run': 'not started', + scanning: 'downloading history', + reconciling: 'rebuilding', + complete: 'up to date', +}; + +/** + * Controls for the offline mail replica (lib/offline-replica/**). + * + * Lives inside the same panel as the search index because they share one + * encrypted file, one key and one purge - presenting them as two unrelated + * features would misrepresent what "delete" deletes. + */ +function OfflineMailSettings({ slot }: { slot: number | undefined }) { + const [status, setStatus] = useState(null); + const [busy, setBusy] = useState(null); + const [message, setMessage] = useState(null); + + const refresh = useCallback(async () => { + setStatus(await fetchReplicaStatus(slot)); + }, [slot]); + + useEffect(() => { void refresh(); }, [refresh]); + + const savePolicy = async (patch: Partial) => { + if (!status) return; + const next: RetentionPolicy = { ...status.policy, ...patch }; + setBusy('policy'); + setMessage(null); + try { + const ok = await updateRetentionPolicy(next, slot); + if (!ok) { setMessage('Could not save the retention setting.'); return; } + // The change is applied by the next cycle - a widen re-scans, a narrow + // evicts - so run one now rather than leaving the number looking wrong. + await chainSync({ slot, max: 2 }); + await refresh(); + } finally { + setBusy(null); + } + }; + + const handleSync = async () => { + setBusy('sync'); + setMessage(null); + try { + const report = await chainSync({ slot }); + if (!report) { setMessage('Offline mail is unavailable on this system.'); return; } + setMessage( + report.ok + ? `Synced ${report.envelopesWritten} messages and ${report.bodiesWritten} bodies.` + + (report.unfinishedWork ? ' More will download in the background.' : '') + + (report.warnings.length > 0 ? ` Notes: ${report.warnings.join('; ')}` : '') + : `Sync failed: ${report.error ?? 'unknown error'}`, + ); + await refresh(); + } finally { + setBusy(null); + } + }; + + const handlePurge = async () => { + setBusy('purge'); + setMessage(null); + try { + const ok = await purgeReplica(slot); + setMessage(ok ? 'Offline mail deleted from this device.' : 'Could not delete offline mail.'); + await refresh(); + } finally { + setBusy(null); + } + }; + + if (!status) return null; + + const stats = status.stats; + const total = stats ? stats.fileBytes : 0; + + return ( + <> + 0 ? ` · ${stats.wantedBodies} still downloading` : '') + : 'Nothing stored yet. Mail downloads automatically as it arrives.' + } + > + {formatBytes(total)} + + + + + + + + + + + + + + + +
+ + +
+
+ + ); +} diff --git a/docs/ELECTRON-OFFLINE-ENGINE-DESIGN.md b/docs/ELECTRON-OFFLINE-ENGINE-DESIGN.md index e84b5ece..b832f07a 100644 --- a/docs/ELECTRON-OFFLINE-ENGINE-DESIGN.md +++ b/docs/ELECTRON-OFFLINE-ENGINE-DESIGN.md @@ -1,3 +1,37 @@ +> # ⚠️ PARTLY REINSTATED — read this note before the SUPERSEDED banner below +> +> A real offline mail replica **now exists**: `lib/offline-replica/**` + +> `app/api/offline/{sync,mail,status}` + `lib/offline-fallback-client.ts`. So the banner below +> ("that scope was dropped") is history, not current state. What was reinstated is the DATA MODEL +> and the SYNC PROTOCOL from this document; what was **not** reinstated is its process +> architecture — and that distinction is the whole reason the review's critical findings did not +> come back with it. +> +> | This document proposed | What was built | +> |---|---| +> | A persistent background worker holding credentials for the process lifetime | **No worker.** A cycle is request-scoped work in an API route using the request's own `jmap_stalwart_ctx` cookie, triggered by the renderer's live push connection — the same shape the search index already uses. | +> | An epoch-fenced multi-account `registry.json` | **No registry, no epochs.** Single-flight per account inside one process; all state in the SQLite file under a real transaction. | +> | Multi-account simultaneous sync | **One request, one account, one cycle**, with hard budgets. | +> | Engine reads a renderer setting to decide whether to sync | **The renderer decides when to sync.** The retention *policy* is durable inside the encrypted store (`/api/offline/status`), because the retention pass genuinely needs it server-side. | +> | An offline read layer with no coherence story for the webmail's local unread-count arithmetic | **The replica is a FALLBACK, never a cache in front of the server** — consulted only after a read has failed at the transport level, so an online session never sees a replica count. This is the answer to review finding H3, the one finding that genuinely returned. | +> +> Consequently: **C2, C3, C4, H1 and H4 remain moot** (they were all consequences of the worker, +> the registry, or a server-side engine reading renderer state), **C1 and H2 remain fixed** by what +> already shipped (`optionalDependencies` + guarded require; the key on an inherited fd), and +> **H3 is now in scope and answered** as above. See `lib/offline-replica/sync.ts`'s header for the +> finding-by-finding version of this table, kept next to the code it constrains. +> +> Two implementation bugs from the mobile client are regressed by name, because both fail silently +> and both cost user data: the **body-tier infinite redownload loop** (durable `gave_up` + +> `shed-by-cap` marks + inserted-not-attempted counting) and the **clock-jump guard that wiped the +> store** (persist the floor that was USED, never the one that was rejected, plus a separate +> `evictionAllowed` bit). Tests: +> `lib/offline-replica/__tests__/{store,retention}.test.ts`, and the real network-cut proof in +> `integration/tests/13-electron-offline-replica.spec.ts`. +> +> Still deliberately out of scope: attachment blobs, offline compose/outbox, delegated/shared +> accounts, and calendar/contacts/files replication. + > # ⚠️ SUPERSEDED — this is not what was built > > This document designs a **full offline mail replica**: a persistent background sync engine with diff --git a/docs/ELECTRON-OFFLINE-ENGINE-REVIEW.md b/docs/ELECTRON-OFFLINE-ENGINE-REVIEW.md index e5460bb6..f5bb4915 100644 --- a/docs/ELECTRON-OFFLINE-ENGINE-REVIEW.md +++ b/docs/ELECTRON-OFFLINE-ENGINE-REVIEW.md @@ -1,3 +1,31 @@ +> # ⚠️ UPDATE — a replica was later built, and this review is why it is shaped the way it is +> +> The table below says most of these findings "stopped existing" because the scope change removed +> the thing they were about. A replica has since been built (`lib/offline-replica/**`), so that +> reasoning was re-examined finding by finding rather than inherited: +> +> - **C1** — still FIXED, and untouched: the replica adds no new dependency and reuses the guarded +> optional require. Both `docker build`s are unaffected. +> - **C2, C3, C4, H1, H4** — still MOOT, and moot *for the same reasons*, because the persistent +> background worker, the shared registry and the server-side-engine-reads-renderer-state shapes +> were **not** reinstated. A cycle is request-scoped work in an API route with no resident +> credential; there is no registry and no epoch; one request syncs one account. Had the worker +> come back, all five would have come back with it. +> - **H2** — still FIXED: the key crosses on an inherited file descriptor, never via environment, +> and is zeroed after each job. The replica reuses that channel rather than inventing a second. +> - **H3 — BACK IN SCOPE, and the only one that is.** This review was right that the webmail does +> local delta arithmetic on mailbox unread counts, and a read-only offline cache underneath it +> needs a coherence story. The answer is an ordering rule: the replica is consulted **only after +> a read has failed at the transport level**, so it is never a cache in front of the server and +> the arithmetic never operates on replica numbers. Enforcing that needed a real signal, because +> `lib/jmap/client.ts` swallows read errors and returns plausible success — hence +> `lib/jmap/transport-health.ts` and the two-part gate in `lib/offline-fallback-client.ts`. +> - The *medium/low* findings (Linux-only API, the vacuous `cipher_version` check, the two bindings +> not being interchangeable) were all already fixed in the shipped index and are inherited. +> +> Nothing in this review turned out to be wrong on re-reading. Its verdict — that the sync-engine +> core transfers and the platform-specific sections were where the danger lay — held exactly. + > # ⚠️ SUPERSEDED — reviews a design that was not built > > This reviews `ELECTRON-OFFLINE-ENGINE-DESIGN.md`, which was **dropped**. Its findings were the diff --git a/integration/tests/13-electron-offline-replica.spec.ts b/integration/tests/13-electron-offline-replica.spec.ts new file mode 100644 index 00000000..aa66ee0c --- /dev/null +++ b/integration/tests/13-electron-offline-replica.spec.ts @@ -0,0 +1,600 @@ +import { test, expect, _electron as electron } from '@playwright/test'; +import type { ElectronApplication, Page } from '@playwright/test'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { createServer } from 'node:net'; +import { get as httpGet } from 'node:http'; +import { createHash, randomBytes } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { ACCOUNTS, JMAP_URL } from './helpers/config'; +import { sendMail } from './helpers/smtp'; +import { JmapClient } from './helpers/jmap'; + +/** + * The offline mail replica (lib/offline-replica/**) against the real Stalwart + * fixture, with a REAL NETWORK CUT. + * + * THE POINT OF THIS FILE: a sync test that never tests the offline case has not + * tested the feature. So test 1 syncs against a live server, then makes the + * backend genuinely unreachable, and only then asserts that a previously-synced + * message still returns its full HTML body - from the encrypted replica, with no + * network available to fall back to. + * + * HOW THE CUT IS MADE. The standalone server is started with + * `JMAP_SERVER_URL` pointing at a LOCAL PROXY that forwards to Stalwart. Killing + * the proxy's listener makes every JMAP request fail with ECONNREFUSED - a real + * transport failure at the socket level, not a mock, not a stubbed fetch, and not + * a flag the code under test can see. Preferred over stopping the Stalwart + * container because it cuts only THIS test's path and leaves the shared fixture + * (and any concurrently-running suite) untouched. + * + * THE SAME TWO CONSTRAINTS as 12-electron-mail-index.spec.ts apply and are why + * this is split into two tests rather than one: + * + * 1. The RENDERER cannot reach this fixture from a production build. It talks + * JMAP directly to Stalwart, which here is deliberately plain HTTP, and the + * production CSP pins `connect-src` to `'self' https: wss:`. NODE_ENV at + * runtime does not help - `next build` inlines it into the middleware. + * 2. The fd-3 key channel cannot survive `next dev`, which claims fd 3 for its + * own IPC. So the two configurations are mutually exclusive: a real key + * channel means no browser, a browser means no key channel. + * + * Test 1 therefore drives the REAL standalone server over HTTP from Node with a + * real fd-3 key channel - no browser needed, because the routes are the thing + * being proven. Test 2 launches the REAL Electron shell to prove the routes exist + * and are reachable in a genuine build, which is the class of failure only a real + * build reveals (the standalone output silently dropping a native prebuild, say). + * + * WHAT THIS FILE DOES NOT PROVE: that `components/email/email-viewer.tsx` paints + * the replica-served body in a browser while offline. That needs a renderer, a + * key channel and a reachable-then-unreachable JMAP server simultaneously, which + * constraints 1 and 2 make impossible against this fixture. The read path returns + * a field-for-field `Email` (asserted below, including `bodyValues` keyed by the + * same partIds as `htmlBody`), and the renderer-side gate is covered by + * `lib/__tests__/offline-fallback-client.test.ts` - but the final paint is NOT + * covered by a real offline browser run. Stated rather than implied. + */ +const alice = ACCOUNTS.alice; +const projectRoot = path.resolve(__dirname, '../..'); + +/** Mirrors lib/mail-index/paths.ts's accountFileToken(). */ +function accountFileToken(accountId: string): string { + return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32); +} + +function getFreePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.unref(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (address && typeof address === 'object') { + const { port } = address; + server.close(() => resolve(port)); + } else { + server.close(() => reject(new Error('Could not allocate a free localhost port'))); + } + }); + }); +} + +function waitForServerReady(url: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + return new Promise((resolve, reject) => { + const attempt = () => { + const req = httpGet(url, (res) => { + res.resume(); + resolve(); + }); + req.on('error', () => { + if (Date.now() > deadline) { + reject(new Error(`Server never became reachable at ${url}`)); + return; + } + setTimeout(attempt, 300); + }); + }; + attempt(); + }); +} + +/** + * A raw TCP forwarder in front of Stalwart, so the test can sever the backend at + * the socket level. `cut()` closes the listener AND destroys every live socket, so + * a pooled keep-alive connection cannot keep working after the cut. + */ +async function startCuttableProxy(target: { host: string; port: number }): Promise<{ + port: number; + cut: () => Promise; + stop: () => Promise; +}> { + const { connect } = await import('node:net'); + const sockets = new Set(); + const server = createServer((incoming) => { + sockets.add(incoming); + incoming.on('close', () => sockets.delete(incoming)); + incoming.on('error', () => incoming.destroy()); + const upstream = connect(target.port, target.host, () => { + incoming.pipe(upstream); + upstream.pipe(incoming); + }); + sockets.add(upstream); + upstream.on('close', () => sockets.delete(upstream)); + upstream.on('error', () => { incoming.destroy(); upstream.destroy(); }); + }); + const port = await getFreePort(); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, '127.0.0.1', () => resolve()); + }); + + const closeAll = () => + new Promise((resolve) => { + for (const s of sockets) s.destroy(); + sockets.clear(); + server.close(() => resolve()); + // `close()` only stops new connections; the destroys above handle the rest. + setTimeout(resolve, 500); + }); + + return { port, cut: closeAll, stop: closeAll }; +} + +/** + * Serves the key protocol of electron/key-service.ts over the child's inherited + * fd. The key and the encryption are real; only safeStorage's wrapping of it is + * out of the picture here, which is what test 2 covers. + */ +function serveKeyChannel(child: ChildProcess, fdIndex: number, key: Buffer): void { + const channel = child.stdio[fdIndex] as NodeJS.ReadWriteStream | null; + if (!channel) throw new Error(`child has no fd ${fdIndex} - key channel missing`); + let buffer = ''; + channel.on('data', (chunk: Buffer) => { + buffer += chunk.toString('utf8'); + let newline: number; + while ((newline = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + if (!line.trim()) continue; + const req = JSON.parse(line) as { id?: number; op?: string }; + const reply = + req.op === 'getIndexKey' + ? { id: req.id, ok: true, key: key.toString('hex') } + : req.op === 'deleteIndexKey' + ? { id: req.id, ok: true } + : { id: req.id, ok: false, code: 'bad-request', error: 'unknown op' }; + channel.write(`${JSON.stringify(reply)}\n`); + } + }); +} + +class Jar { + private cookies = new Map(); + absorb(response: Response): void { + for (const raw of response.headers.getSetCookie()) { + const [pair] = raw.split(';'); + const eq = pair.indexOf('='); + if (eq <= 0) continue; + this.cookies.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim()); + } + } + header(): string { + return [...this.cookies].map(([k, v]) => `${k}=${v}`).join('; '); + } +} + +interface CycleReport { + ok: boolean; + unfinishedWork: boolean; + bootstrapped: boolean; + envelopesWritten: number; + bodiesWritten: number; + envelopesDeleted: number; + coveragePhase: string; + resyncRequired: boolean; + warnings: string[]; + error?: string; + errorClass?: string; +} + +test.describe('Electron desktop shell - offline mail replica', () => { + test('syncs full bodies, then serves a synced message with the backend UNREACHABLE', async () => { + test.setTimeout(240_000); + + const jmap = await JmapClient.connect(alice.email, alice.password); + await jmap.reset(); + + const stamp = Date.now(); + const subject = `IT replica subject ${stamp}`; + // Appears ONLY in the HTML body, so a hit proves the full body was stored - + // not the preview or the subject, which any envelope already carries. + const bodyPhrase = `luzernrenewal${stamp}`; + const htmlMarker = `${bodyPhrase}`; + + await sendMail({ + from: alice.email, + authPass: alice.password, + to: alice.email, + subject, + body: `plain text ${bodyPhrase}`, + html: `

Please review the ${htmlMarker} before September.

`, + }); + // A second message, so "the list came from the replica" is not a one-row + // coincidence. + await sendMail({ + from: alice.email, + authPass: alice.password, + to: alice.email, + subject: `IT replica second ${stamp}`, + body: 'the second message', + }); + + const storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-it-')); + const key = randomBytes(32); + const stalwart = new URL(JMAP_URL); + const proxy = await startCuttableProxy({ + host: stalwart.hostname, + port: Number(stalwart.port || 80), + }); + const proxiedJmapUrl = `http://127.0.0.1:${proxy.port}`; + + const port = await getFreePort(); + const baseUrl = `http://127.0.0.1:${port}`; + const serverEntry = path.join(projectRoot, '.next', 'standalone', 'server.js'); + expect( + fs.existsSync(serverEntry), + `missing ${serverEntry} - run "npm run build:standalone" first`, + ).toBe(true); + + const server = spawn(process.execPath, [serverEntry], { + cwd: path.dirname(serverEntry), + env: { + ...process.env, + PORT: String(port), + HOSTNAME: '127.0.0.1', + NODE_ENV: 'production', + // Through the cuttable proxy, so the backend can be severed later. + JMAP_SERVER_URL: proxiedJmapUrl, + SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!', + VNCMAIL_DESKTOP_STORE_DIR: storeDir, + VNCMAIL_DESKTOP_KEY_FD: '3', + }, + stdio: ['pipe', 'pipe', 'pipe', 'pipe'], + }); + server.stderr?.on('data', (chunk) => process.stderr.write(`[standalone] ${chunk}`)); + serveKeyChannel(server, 3, key); + + const jar = new Jar(); + const call = async (url: string, init?: RequestInit): Promise => { + const response = await fetch(`${baseUrl}${url}`, { + ...init, + headers: { ...(init?.headers ?? {}), cookie: jar.header() }, + }); + jar.absorb(response); + return response; + }; + const sync = async (body: Record = {}): Promise => { + const response = await call('/api/offline/sync', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + const parsed = await response.json(); + expect(response.status, JSON.stringify(parsed)).toBe(200); + return parsed.report as CycleReport; + }; + + try { + await waitForServerReady(baseUrl, 90_000); + + const login = await call('/api/auth/session?slot=0', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + serverUrl: proxiedJmapUrl, + username: alice.email, + password: alice.password, + slot: 0, + }), + }); + const loginText = await login.text(); + expect(login.status, `login failed: ${loginText}`).toBe(200); + + // The gate must be open and the native binding loaded, or every assertion + // below would fail for an unrelated reason. + const reachable = await call('/api/offline/status'); + const reachableText = await reachable.text(); + expect( + reachable.status, + `replica routes unreachable: ${reachableText.slice(0, 400)}`, + ).toBe(200); + + // ── ONLINE: bootstrap, then chain until the cycle reports itself done ── + const first = await sync(); + expect(first.error, `first cycle failed: ${first.error}`).toBeUndefined(); + expect(first.bootstrapped, 'the first cycle must bootstrap').toBe(true); + + let report = first; + for (let i = 0; i < 12 && report.unfinishedWork; i++) report = await sync(); + expect( + report.unfinishedWork, + `sync never settled: ${JSON.stringify(report)}`, + ).toBe(false); + // Termination is a real property here: the body-queue give-up marks and the + // inserted-not-attempted count are what stop this looping forever. + expect(report.coveragePhase).toBe('complete'); + expect(report.resyncRequired).toBe(false); + + const status = await (await call('/api/offline/status')).json(); + expect(status.synced).toBe(true); + expect( + status.stats.envelopes, + `no envelopes stored: ${JSON.stringify(status.stats)}`, + ).toBeGreaterThanOrEqual(2); + expect( + status.stats.bodies, + `no BODIES stored - the replica would be no better than the search index`, + ).toBeGreaterThanOrEqual(2); + expect(status.stats.mailboxes).toBeGreaterThan(0); + + // ── THE DELTA PATH: a message that arrives AFTER the cursor was captured ── + // Bootstrap alone would satisfy every assertion below, so this is what actually + // exercises `Email/changes` and proves the stored cursor is USABLE rather than + // merely present. It is also the assertion that fails if an `Email/get` state + // token is ever adopted as a `/changes` cursor: the fast-forwarded cursor + // reports no changes, and this message never arrives. + const deltaSubject = `IT replica delta ${stamp}`; + const deltaPhrase = `bernrenewal${stamp}`; + await sendMail({ + from: alice.email, + authPass: alice.password, + to: alice.email, + subject: deltaSubject, + body: `plain ${deltaPhrase}`, + html: `

delta ${deltaPhrase}

`, + }); + + let delta = await sync(); + for (let i = 0; i < 10 && (delta.unfinishedWork || delta.envelopesWritten === 0); i++) { + delta = await sync(); + } + expect(delta.bootstrapped, 'the delta cycle must NOT re-bootstrap').toBe(false); + const afterDelta = await (await call('/api/offline/status')).json(); + expect( + afterDelta.stats.envelopes, + `Email/changes did not deliver a message that arrived after the cursor was ` + + `captured: ${JSON.stringify(afterDelta.stats)}`, + ).toBeGreaterThanOrEqual(3); + expect( + afterDelta.stats.bodies, + 'the delta path delivered the envelope but never queued its body', + ).toBeGreaterThanOrEqual(3); + expect(afterDelta.resyncRequired, 'a healthy delta cycle must not invalidate a cursor').toBe(false); + + // Find the message and its mailbox while still online, so the offline phase + // asserts on known ids rather than discovering them from the thing under test. + const mailboxesOnline = await (await call('/api/offline/mail?kind=mailboxes')).json(); + const inbox = (mailboxesOnline.mailboxes as Array<{ id: string; role?: string }>) + .find((m) => m.role === 'inbox'); + expect(inbox, 'the replica holds no inbox').toBeTruthy(); + + const listOnline = await ( + await call(`/api/offline/mail?kind=list&mailboxId=${encodeURIComponent(inbox!.id)}&limit=50`) + ).json(); + const target = (listOnline.emails as Array<{ id: string; subject?: string }>) + .find((e) => e.subject === subject); + expect(target, `the synced message is not in the replica: ${JSON.stringify(listOnline.emails?.map((e: {subject?: string}) => e.subject))}`).toBeTruthy(); + + // ── THE CUT: sever the backend at the socket level ──────────────────── + await proxy.cut(); + + // Prove the cut is real, from inside the server process's own network + // namespace: a live JMAP call must now fail. `/api/offline/sync` reaches + // Stalwart first thing, so it is the honest probe. + const afterCut = await call('/api/offline/sync', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + const afterCutBody = await afterCut.json(); + expect( + afterCut.status, + `the backend is still reachable, so the offline assertions below would prove nothing: ` + + `${JSON.stringify(afterCutBody)}`, + ).not.toBe(200); + + // ── OFFLINE: the actual feature ─────────────────────────────────────── + const messageResponse = await call( + `/api/offline/mail?kind=message&id=${encodeURIComponent(target!.id)}`, + ); + // Read the body ONCE: `expect`'s message argument is evaluated eagerly, so + // putting `await response.text()` in it consumes the stream before .json(). + const messageText = await messageResponse.text(); + expect( + messageResponse.status, + `the offline read path failed with the backend down: ${messageText.slice(0, 400)}`, + ).toBe(200); + const offline = JSON.parse(messageText); + expect(offline.available).toBe(true); + expect(offline.hasBody, 'the message has no stored body offline').toBe(true); + + const email = offline.email as { + id: string; subject?: string; receivedAt: string; + htmlBody?: Array<{ partId: string; type: string }>; + textBody?: Array<{ partId: string }>; + bodyValues?: Record; + from?: Array<{ email: string }>; + keywords?: Record; + mailboxIds?: Record; + headers?: Record; + }; + + expect(email.id).toBe(target!.id); + expect(email.subject).toBe(subject); + + // THE ASSERTION: the full HTML body, recovered with no network. + const htmlPartId = email.htmlBody?.[0]?.partId; + expect(htmlPartId, 'no htmlBody part offline').toBeTruthy(); + const html = email.bodyValues?.[htmlPartId as string]?.value ?? ''; + expect( + html, + 'the HTML body is not in the replica - this is the whole feature', + ).toContain(htmlMarker); + expect(html).toContain(bodyPhrase); + + // `bodyValues` MUST be keyed by the same partIds as htmlBody/textBody, or + // email-viewer.tsx's isBodyLoading gate sits on its skeleton forever + // (hasBodyParts true, bodyValues unusable). + for (const part of [...(email.htmlBody ?? []), ...(email.textBody ?? [])]) { + expect( + email.bodyValues?.[part.partId], + `bodyValues is missing partId ${part.partId}, which the viewer requires`, + ).toBeTruthy(); + } + + // The rest of the shape the renderer reads. + expect(email.from?.[0]?.email).toBe(alice.email); + expect(email.receivedAt).toBeTruthy(); + expect(Object.keys(email.mailboxIds ?? {})).toContain(inbox!.id); + // Header normalisation happened server-side (the array -> record flattening + // the online path does in parseEmailHeaders). + expect(email.headers && !Array.isArray(email.headers)).toBe(true); + + // The list and the folder tree must also survive the cut. + const listOffline = await ( + await call(`/api/offline/mail?kind=list&mailboxId=${encodeURIComponent(inbox!.id)}&limit=50`) + ).json(); + expect(listOffline.available).toBe(true); + expect(listOffline.emails.length).toBeGreaterThanOrEqual(2); + expect( + (listOffline.emails as Array<{ subject?: string }>).map((e) => e.subject), + ).toContain(subject); + + const mailboxesOffline = await (await call('/api/offline/mail?kind=mailboxes')).json(); + expect(mailboxesOffline.available).toBe(true); + expect((mailboxesOffline.mailboxes as unknown[]).length).toBeGreaterThan(0); + + // Status must be readable offline too - a user with no network still needs + // to see what they have and be able to free the space. + const statusOffline = await (await call('/api/offline/status')).json(); + expect(statusOffline.ok).toBe(true); + expect(statusOffline.stats.bodies).toBeGreaterThanOrEqual(2); + + // A cycle attempted while offline must classify as Transport and must NOT + // touch the data. "Offline is not an error." + expect( + ['Transport', 'ServerTransient'].includes(String(afterCutBody.code)), + `an offline cycle must classify as Transport/ServerTransient so the caller retries ` + + `rather than treating the feature as broken; got code=${afterCutBody.code} ` + + `status=${afterCut.status} body=${JSON.stringify(afterCutBody)}`, + ).toBe(true); + const afterOfflineCycle = await (await call('/api/offline/status')).json(); + expect( + afterOfflineCycle.stats.envelopes, + 'an offline cycle deleted data - a transport failure must never do that', + ).toBe(statusOffline.stats.envelopes); + expect(afterOfflineCycle.resyncRequired).toBe(false); + + // ── PURGE: the retention control has to actually free the space ──────── + const purge = await call('/api/offline/status', { method: 'DELETE' }); + expect(purge.status).toBe(200); + const purged = await (await call('/api/offline/status')).json(); + expect(purged.synced).toBe(false); + expect(purged.coveragePhase).toBe('never-run'); + } finally { + server.kill(); + await proxy.stop(); + // Let the process release its WAL files before reading them. + await new Promise((r) => setTimeout(r, 700)); + } + + // ── the file on disk is genuinely encrypted ───────────────────────────── + const accountId = `${alice.email}@127.0.0.1`; + const dbPath = path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`); + expect(fs.existsSync(dbPath), `no replica database at ${dbPath}`).toBe(true); + + const onDisk = Buffer.concat( + ['', '-wal', '-shm'] + .map((suffix) => `${dbPath}${suffix}`) + .filter((f) => fs.existsSync(f)) + .map((f) => fs.readFileSync(f)), + ); + expect(onDisk.length).toBeGreaterThan(0); + // `PRAGMA key` is a silent no-op on a non-SQLCipher binding - no error, a + // working database, and the mail in cleartext - so every functional assertion + // above would pass either way. These are the ones that catch it. + expect( + fs.readFileSync(dbPath).subarray(0, 15).toString('latin1'), + 'the replica file has a plain SQLite header - it is NOT encrypted', + ).not.toBe('SQLite format 3'); + expect( + onDisk.includes(bodyPhrase), + 'the message body is recoverable from the raw database bytes - not encrypted', + ).toBe(false); + expect( + onDisk.includes(subject), + 'the subject is recoverable from the raw database bytes - not encrypted', + ).toBe(false); + + fs.rmSync(storeDir, { recursive: true, force: true }); + }); + + test('wiring: the real standalone boot reaches the replica routes with a real safeStorage key', async () => { + test.setTimeout(180_000); + // A FRESH profile is load-bearing, not hygiene: the 401 asserted below is + // "nobody is signed in", and a leftover jmap_stalwart_ctx cookie from any + // previous run turns it into a 200. + const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-wiring-')); + const electronApp: ElectronApplication = await electron.launch({ + args: [projectRoot, `--user-data-dir=${userDataDir}`], + env: { + ...process.env, + JMAP_SERVER_URL: JMAP_URL, + SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!', + }, + }); + + try { + const appWindow: Page = await electronApp.firstWindow(); + await appWindow.waitForLoadState('domcontentloaded'); + await appWindow.locator('#username').waitFor({ state: 'visible', timeout: 90_000 }); + + const encryptionAvailable = await electronApp.evaluate(({ safeStorage }) => + safeStorage.isEncryptionAvailable(), + ); + expect( + encryptionAvailable, + 'safeStorage reports no encryption available, so main.ts correctly disabled the ' + + 'feature - this assertion cannot pass here', + ).toBe(true); + + // 401 = the gate opened, the native binding loaded from the REAL standalone + // artifact, and the fd-3 key channel is present; it refuses only because + // nobody is signed in. + // 404 => VNCMAIL_DESKTOP_STORE_DIR was never set + // 503 => the native binding or the key channel is missing from the real + // build - the class of failure only a real build reveals + for (const route of [ + '/api/offline/status', + '/api/offline/mail?kind=mailboxes', + '/api/offline/sync', + ]) { + const probe = await appWindow.evaluate(async (url) => { + const response = await fetch(url, { + method: url.endsWith('/sync') ? 'POST' : 'GET', + }); + return { status: response.status, body: (await response.text()).slice(0, 300) }; + }, route); + expect( + probe.status, + `${route}: expected 401 (reachable, unauthenticated) but got ${probe.status}: ${probe.body}`, + ).toBe(401); + } + } finally { + await electronApp.close(); + fs.rmSync(userDataDir, { recursive: true, force: true }); + } + }); +}); diff --git a/integration/tests/helpers/smtp.ts b/integration/tests/helpers/smtp.ts index 734c2e00..e7b61387 100644 --- a/integration/tests/helpers/smtp.ts +++ b/integration/tests/helpers/smtp.ts @@ -22,6 +22,14 @@ interface SendOptions { subject: string; /** Plain-text body. */ body: string; + /** + * Optional HTML alternative, sent as multipart/alternative alongside `body`. + * + * Added for 13-electron-offline-replica.spec.ts, which has to prove the offline + * replica stores a real HTML body and not just the plain-text excerpt the search + * index keeps - so the message needs a genuine distinct text/html part. + */ + html?: string; /** Extra headers (e.g. custom Message-ID / In-Reply-To for threading). */ headers?: Record; /** Optional single attachment (sent as multipart/mixed, base64). */ @@ -158,6 +166,22 @@ export async function sendMail(opts: SendOptions): Promise { b64, `--${boundary}--`, ].join('\r\n'); + } else if (opts.html) { + const boundary = 'italt_boundary_0001'; + headers['MIME-Version'] = '1.0'; + headers['Content-Type'] = `multipart/alternative; boundary="${boundary}"`; + // text first, html second: multipart/alternative is least-to-most preferred. + mime = [ + `--${boundary}`, + 'Content-Type: text/plain; charset=utf-8', + '', + crlf(opts.body), + `--${boundary}`, + 'Content-Type: text/html; charset=utf-8', + '', + crlf(opts.html), + `--${boundary}--`, + ].join('\r\n'); } else { headers['Content-Type'] = 'text/plain; charset=utf-8'; mime = crlf(opts.body); diff --git a/lib/__tests__/offline-fallback-client.test.ts b/lib/__tests__/offline-fallback-client.test.ts new file mode 100644 index 00000000..cc5c6eb4 --- /dev/null +++ b/lib/__tests__/offline-fallback-client.test.ts @@ -0,0 +1,224 @@ +// The two-part fallback gate. +// +// `lib/jmap/client.ts`'s read methods swallow their own errors and return +// plausible success, so a "looks empty" result is NOT evidence of a network +// failure - it is also what a genuinely empty folder returns, and +// `getMailboxes()` fabricates a synthetic Inbox rather than throwing. Falling back +// on the shape alone would serve stale replica rows over a folder the user had +// just emptied. So the gate is: suspicious result AND a `fetch` rejection recorded +// during that exact call. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { IJMAPClient } from '@/lib/jmap/client-interface'; +import type { Email, Mailbox } from '@/lib/jmap/types'; +import { noteTransportFailure, resetTransportHealth } from '@/lib/jmap/transport-health'; + +const readOfflineMailboxes = vi.fn(); +const readOfflineList = vi.fn(); +const readOfflineMessage = vi.fn(); +const isReplicaUnavailable = vi.fn(() => false); + +vi.mock('@/lib/offline-replica-client', () => ({ + readOfflineMailboxes: (...a: unknown[]) => readOfflineMailboxes(...a), + readOfflineList: (...a: unknown[]) => readOfflineList(...a), + readOfflineMessage: (...a: unknown[]) => readOfflineMessage(...a), + isReplicaUnavailable: () => isReplicaUnavailable(), +})); + +vi.mock('@/stores/account-store', () => ({ + useAccountStore: { + getState: () => ({ + accounts: [{ id: 'alice@mail.example.org', cookieSlot: 3, serverIdentifiers: [] }], + }), + }, +})); + +const { withOfflineFallback } = await import('@/lib/offline-fallback-client'); + +function replicaEmail(id: string): Email { + return { + id, threadId: 't', mailboxIds: { inbox: true }, keywords: {}, size: 1, + receivedAt: '2026-08-01T00:00:00.000Z', hasAttachment: false, + htmlBody: [{ partId: '1', blobId: 'b', size: 1, type: 'text/html' }], + bodyValues: { '1': { value: '

from the replica

' } }, + }; +} + +interface Stub extends Partial { + getEmail: IJMAPClient['getEmail']; + getEmails: IJMAPClient['getEmails']; + getMailboxes: IJMAPClient['getMailboxes']; + getAllMailboxes: IJMAPClient['getAllMailboxes']; +} + +/** Reproduces the client's real error-swallowing shapes. */ +function stubClient(overrides: Partial = {}): IJMAPClient { + const stub = { + getUsername: () => 'alice', + getServerUrl: () => 'https://mail.example.org', + getAccountId: () => 'primary', + getEmail: async () => null, + getEmails: async () => ({ emails: [] as Email[], hasMore: false, total: 0 }), + getMailboxes: async () => ([ + // The exact placeholder client.ts fabricates on failure. + { id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0, + unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true, + myRights: {} } as unknown as Mailbox, + ]), + getAllMailboxes: async () => ([ + { id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0, + unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true, + myRights: {} } as unknown as Mailbox, + ]), + ...overrides, + }; + return stub as unknown as IJMAPClient; +} + +describe('withOfflineFallback', () => { + beforeEach(() => { + resetTransportHealth(); + vi.clearAllMocks(); + isReplicaUnavailable.mockReturnValue(false); + }); + + it('does NOT consult the replica when the server answered "empty"', async () => { + // The whole point. An empty folder must render empty, not as whatever the + // replica last held. + const client = withOfflineFallback(stubClient()); + const result = await client.getEmails('inbox'); + expect(result.emails).toEqual([]); + expect(readOfflineList).not.toHaveBeenCalled(); + + expect(await client.getEmail('e1')).toBeNull(); + expect(readOfflineMessage).not.toHaveBeenCalled(); + }); + + it('consults the replica when a transport failure happened DURING the call', async () => { + readOfflineList.mockResolvedValue({ + emails: [replicaEmail('e1')], total: 1, hasMore: false, + }); + const client = withOfflineFallback( + stubClient({ + getEmails: async () => { + // What authenticatedFetch does when `fetch` rejects. + noteTransportFailure(); + return { emails: [], hasMore: false, total: 0 }; + }, + }), + ); + const result = await client.getEmails('inbox', undefined, 25, 0); + expect(result.emails.map((e) => e.id)).toEqual(['e1']); + expect(result.total).toBe(1); + // And it asks for the right slot, so a multi-account shell reads the right file. + expect(readOfflineList).toHaveBeenCalledWith('inbox', { limit: 25, offset: 0, slot: 3 }); + }); + + it('ignores a stale transport failure from BEFORE the call', async () => { + // The counter is sampled per call precisely so an old failure cannot make a + // later successful-but-empty read look offline. + noteTransportFailure(); + const client = withOfflineFallback(stubClient()); + await client.getEmails('inbox'); + expect(readOfflineList).not.toHaveBeenCalled(); + }); + + it('serves a full message from the replica, but refuses an envelope-only hit', async () => { + // An envelope with no bodyValues would render blank AND leave the viewer's + // isBodyLoading gate stuck on its skeleton, which is worse than saying the + // message is unavailable. + readOfflineMessage.mockResolvedValue({ email: replicaEmail('e1'), hasBody: true }); + const client = withOfflineFallback( + stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }), + ); + const email = await client.getEmail('e1'); + expect(email?.bodyValues?.['1'].value).toContain('from the replica'); + + resetTransportHealth(); + readOfflineMessage.mockResolvedValue({ email: replicaEmail('e2'), hasBody: false }); + const client2 = withOfflineFallback( + stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }), + ); + expect(await client2.getEmail('e2')).toBeNull(); + }); + + it('recognises the synthetic Inbox placeholder and replaces it', async () => { + readOfflineMailboxes.mockResolvedValue([ + { id: 'mb1', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 9, unreadEmails: 2, + totalThreads: 9, unreadThreads: 2, isSubscribed: true, myRights: {} } as unknown as Mailbox, + ]); + const client = withOfflineFallback( + stubClient({ + getAllMailboxes: async () => { + noteTransportFailure(); + return [ + { id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0, + unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true, + myRights: {} } as unknown as Mailbox, + ]; + }, + }), + ); + const mailboxes = await client.getAllMailboxes(); + expect(mailboxes.map((m) => m.id)).toEqual(['mb1']); + }); + + it('keeps a REAL single-mailbox server result even after a transport failure', async () => { + // A genuine server that happens to return one inbox has a real id and real + // counts; only the exact placeholder shape may be replaced. + const real = { + id: 'real-inbox-id', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 12, + unreadEmails: 1, totalThreads: 12, unreadThreads: 1, isSubscribed: true, myRights: {}, + } as unknown as Mailbox; + const client = withOfflineFallback( + stubClient({ getAllMailboxes: async () => { noteTransportFailure(); return [real]; } }), + ); + expect((await client.getAllMailboxes())[0].id).toBe('real-inbox-id'); + expect(readOfflineMailboxes).not.toHaveBeenCalled(); + }); + + it('never answers a read scoped to a delegated account', async () => { + // v1 replicates the PRIMARY mail account only, so the replica has no rows for + // a shared account and answering "empty" would be worse than the client's own. + const client = withOfflineFallback( + stubClient({ + getEmails: async () => { noteTransportFailure(); return { emails: [], hasMore: false, total: 0 }; }, + }), + ); + await client.getEmails('inbox', 'someone-elses-account'); + expect(readOfflineList).not.toHaveBeenCalled(); + }); + + it('never answers a keyword- or category-filtered query', async () => { + // Those are server-side queries the replica does not reproduce. Serving an + // unfiltered page in their place would silently show the wrong set. + const failing = async () => { noteTransportFailure(); return { emails: [], hasMore: false, total: 0 }; }; + const c1 = withOfflineFallback(stubClient({ getEmails: failing })); + await c1.getEmails('inbox', undefined, 25, 0, '$flagged'); + expect(readOfflineList).not.toHaveBeenCalled(); + + resetTransportHealth(); + const c2 = withOfflineFallback(stubClient({ getEmails: failing })); + await c2.getEmails('inbox', undefined, 25, 0, undefined, true, { from: 'x' }); + expect(readOfflineList).not.toHaveBeenCalled(); + }); + + it('stops asking once the replica reports itself absent', async () => { + isReplicaUnavailable.mockReturnValue(true); + const client = withOfflineFallback( + stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }), + ); + expect(await client.getEmail('e1')).toBeNull(); + expect(readOfflineMessage).not.toHaveBeenCalled(); + }); + + it('is idempotent, so re-wrapping a client does not stack fallbacks', async () => { + readOfflineMessage.mockResolvedValue({ email: replicaEmail('e1'), hasBody: true }); + const base = stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }); + const once = withOfflineFallback(base); + const twice = withOfflineFallback(once); + expect(twice).toBe(once); + await twice.getEmail('e1'); + expect(readOfflineMessage).toHaveBeenCalledTimes(1); + }); +}); diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index 16502ccd..0f06a833 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -3,6 +3,7 @@ import type { SieveScript, SieveCapabilities } from "./sieve-types"; import type { IJMAPClient } from "./client-interface"; import { toWildcardQuery } from "./search-utils"; import { batched, itemsPerRequest } from "./request-limits"; +import { noteTransportFailure, noteTransportSuccess } from "./transport-health"; import { debug } from "@/lib/debug"; import { normalizeCalendarEventLike } from "@/lib/calendar-event-normalization"; @@ -683,11 +684,24 @@ export class JMAPClient implements IJMAPClient { try { response = await fetch(url, { ...init, headers }); } catch (error) { + // A `fetch` REJECTION - and only that - is a transport failure. Recorded so + // the offline replica's read fallback can tell "the network is down" from + // "the folder is empty", which the error-swallowing in getEmails/getEmail/ + // getMailboxes otherwise makes indistinguishable (see + // lib/jmap/transport-health.ts). Deliberately NOT recorded for a 4xx/5xx or + // a 429: in those cases the server answered, so it is reachable. + noteTransportFailure(); // Network error: retry once after brief delay (transient proxy/connection issues) if (this.reconnecting) throw error; await new Promise(r => setTimeout(r, 1000)); - response = await fetch(url, { ...init, headers }); + try { + response = await fetch(url, { ...init, headers }); + } catch (retryError) { + noteTransportFailure(); + throw retryError; + } } + noteTransportSuccess(); // Handle 429 rate limiting - stop immediately, do not retry if (response.status === 429) { diff --git a/lib/jmap/transport-health.ts b/lib/jmap/transport-health.ts new file mode 100644 index 00000000..6b47ce9a --- /dev/null +++ b/lib/jmap/transport-health.ts @@ -0,0 +1,62 @@ +// A single monotonic counter of JMAP TRANSPORT failures. +// +// WHY THIS EXISTS. The offline replica is a read-path FALLBACK, and to be one it +// has to know that a read genuinely failed. `lib/jmap/client.ts` makes that +// impossible to see from the outside: its read methods swallow their own errors +// and return plausible-looking success. `getEmails()` returns +// `{ emails: [], hasMore: false, total: 0 }`, so a dead network is +// indistinguishable from an empty folder. `getEmail()` returns `null`. +// `getMailboxes()` returns a SYNTHETIC single Inbox. Falling back on those shapes +// alone would mean serving stale replica rows for a folder the user had genuinely +// just emptied. +// +// So `authenticatedFetch` bumps this counter when, and only when, `fetch` itself +// rejects - not on a 4xx, not on a 429 (that is a rate limit, and the server is +// plainly reachable), not on a JMAP method error. The fallback layer samples the +// counter before and after a call: a suspicious result PLUS an increment during +// that exact call is a transport failure. Either signal alone is not enough. +// +// Module-level rather than per-client on purpose: it answers "is the network +// working right now", which is a property of the machine, not of one account's +// client instance. + +let failures = 0; +let lastFailureAt = 0; +let lastSuccessAt = 0; + +/** Called only when `fetch` itself rejects. Never for an HTTP status. */ +export function noteTransportFailure(): void { + failures++; + lastFailureAt = Date.now(); +} + +export function noteTransportSuccess(): void { + lastSuccessAt = Date.now(); +} + +/** Monotonic. Sample before and after a call to attribute a failure to it. */ +export function transportFailureCount(): number { + return failures; +} + +export function transportHealth(): { + failures: number; + lastFailureAt: number; + lastSuccessAt: number; + /** Best-effort "probably offline": a failure more recent than any success. */ + likelyOffline: boolean; +} { + return { + failures, + lastFailureAt, + lastSuccessAt, + likelyOffline: lastFailureAt > lastSuccessAt, + }; +} + +/** Test-only reset. */ +export function resetTransportHealth(): void { + failures = 0; + lastFailureAt = 0; + lastSuccessAt = 0; +} diff --git a/lib/mail-index/store.ts b/lib/mail-index/store.ts index 4e89793d..cae2a6f8 100644 --- a/lib/mail-index/store.ts +++ b/lib/mail-index/store.ts @@ -178,6 +178,12 @@ export class MailIndex { try { db.pragma('journal_mode = WAL'); db.pragma('synchronous = NORMAL'); + // The offline replica (lib/offline-replica/**) is a SECOND connection to + // this same file, writing disjoint tables. WAL lets a writer and readers + // coexist, but two WRITERS get SQLITE_BUSY immediately without this - and + // both subsystems are driven by the same renderer push handler, so they + // genuinely do overlap. + db.pragma('busy_timeout = 8000'); version = readSchemaVersion(db); } catch { db.close(); @@ -189,6 +195,12 @@ export class MailIndex { assertEncrypted(db, dbPath); db.pragma('journal_mode = WAL'); db.pragma('synchronous = NORMAL'); + // The offline replica (lib/offline-replica/**) is a SECOND connection to + // this same file, writing disjoint tables. WAL lets a writer and readers + // coexist, but two WRITERS get SQLITE_BUSY immediately without this - and + // both subsystems are driven by the same renderer push handler, so they + // genuinely do overlap. + db.pragma('busy_timeout = 8000'); version = null; } diff --git a/lib/offline-fallback-client.ts b/lib/offline-fallback-client.ts new file mode 100644 index 00000000..095e25a5 --- /dev/null +++ b/lib/offline-fallback-client.ts @@ -0,0 +1,180 @@ +// The read-path fallback. Wraps an `IJMAPClient` so that when a mail read fails +// because the network is down, the answer comes from the encrypted offline replica +// instead of an empty list. +// +// WHY THIS SHAPE, AND NOT A CACHE. The replica is consulted ONLY after a read has +// genuinely failed at the transport level. That ordering is the whole coherence +// story for the design review's H3: the webmail does local delta arithmetic on +// mailbox unread counts for mark-read/move/delete, and if the replica sat in FRONT +// of the server that arithmetic would operate on replica numbers and need +// reconciliation rules. Behind the server, an online session never sees a replica +// value at all, and while offline any count drift is bounded and repaired by the +// next `Mailbox/changes`. +// +// WHY IT IS NOT ENOUGH TO LOOK AT THE RESULT. `lib/jmap/client.ts`'s read methods +// swallow their own errors and return plausible success: `getEmails()` returns an +// empty page, `getEmail()` returns `null`, `getMailboxes()` returns a SYNTHETIC +// single Inbox. Falling back on those shapes alone would serve stale replica rows +// for a folder the user had genuinely just emptied. So the test is TWO-PART: a +// suspicious result AND a `fetch` rejection recorded during that exact call +// (`lib/jmap/transport-health.ts`). A 4xx, a 429 or a JMAP method error all mean +// the server answered, so none of them triggers a fallback. +// +// Mutates the instance rather than wrapping it in a Proxy: `JMAPClient` is a large +// class whose methods call each other through `this`, and instance patching keeps +// `this` identity exactly as it was. Idempotent, so re-wrapping the same client is +// harmless. + +import { generateAccountId } from '@/lib/account-utils'; +import type { IJMAPClient } from '@/lib/jmap/client-interface'; +import type { Email, Mailbox } from '@/lib/jmap/types'; +import { transportFailureCount } from '@/lib/jmap/transport-health'; +import { + isReplicaUnavailable, readOfflineList, readOfflineMailboxes, readOfflineMessage, +} from '@/lib/offline-replica-client'; + +const WRAPPED = Symbol.for('vncmail.offlineFallback.wrapped'); + +/** + * A `getMailboxes()` result that is really the client's offline placeholder. + * + * `client.ts` fabricates exactly this on failure: one mailbox, id `INBOX`, role + * `inbox`, zero counts. Matching it precisely matters - a real server that happens + * to return a single inbox has a real id and real counts. + */ +function isSyntheticMailboxList(mailboxes: readonly Mailbox[]): boolean { + return ( + mailboxes.length === 1 && + mailboxes[0]?.id === 'INBOX' && + mailboxes[0]?.totalEmails === 0 && + mailboxes[0]?.unreadEmails === 0 + ); +} + +/** Resolves this client's cookie slot, so a multi-account shell reads the right replica. */ +async function slotFor(client: IJMAPClient): Promise { + try { + const { useAccountStore } = await import('@/stores/account-store'); + const id = generateAccountId(client.getUsername(), client.getServerUrl()); + const accounts = useAccountStore.getState().accounts; + const match = + accounts.find((a) => a.id === id) ?? + accounts.find((a) => a.serverIdentifiers?.includes(id)); + return match?.cookieSlot; + } catch { + return undefined; + } +} + +/** + * True when the replica may answer for this call. + * + * v1 replicates the PRIMARY mail account only, so a read explicitly scoped to a + * delegated/shared account must never be answered from it - the replica simply has + * no rows, and answering "empty" would be worse than the client's own empty. + */ +function scopedToPrimary(client: IJMAPClient, accountId?: string): boolean { + if (!accountId) return true; + try { + return accountId === client.getAccountId(); + } catch { + return false; + } +} + +export function withOfflineFallback(client: T): T { + const flagged = client as unknown as Record; + if (flagged[WRAPPED]) return client; + flagged[WRAPPED] = true; + + const target = client as unknown as IJMAPClient; + const originalGetEmail = target.getEmail.bind(target); + const originalGetEmails = target.getEmails.bind(target); + const originalGetMailboxes = target.getMailboxes.bind(target); + const originalGetAllMailboxes = target.getAllMailboxes.bind(target); + + target.getEmail = async (emailId: string, accountId?: string): Promise => { + const before = transportFailureCount(); + const online = await originalGetEmail(emailId, accountId); + if (online) return online; + if (isReplicaUnavailable()) return online; + // `null` alone is ambiguous: it is also what a genuinely-missing id returns. + // Only a transport failure during THIS call earns a fallback. + if (transportFailureCount() === before) return online; + if (!scopedToPrimary(client, accountId)) return online; + + const offline = await readOfflineMessage(emailId, await slotFor(client)); + // An envelope with no body would render blank AND leave the viewer's + // `isBodyLoading` gate stuck, so it is not an answer - better to keep the + // client's `null` and let the UI say the message is unavailable offline. + if (!offline?.email || !offline.hasBody) return online; + return offline.email; + }; + + target.getEmails = async ( + mailboxId?: string, + accountId?: string, + limit: number = 50, + position: number = 0, + hasKeyword?: string, + pinnedFirst?: boolean, + extraFilter?: Record, + ): Promise<{ emails: Email[]; hasMore: boolean; total: number }> => { + const before = transportFailureCount(); + const online = await originalGetEmails( + mailboxId, accountId, limit, position, hasKeyword, pinnedFirst, extraFilter, + ); + if (online.emails.length > 0) return online; + if (isReplicaUnavailable()) return online; + if (transportFailureCount() === before) return online; + if (!scopedToPrimary(client, accountId)) return online; + // A keyword or category filter is a server-side query the replica does not + // reproduce. Serving an unfiltered page in its place would silently show the + // wrong set, which is worse than showing nothing. + if (hasKeyword || extraFilter) return online; + + const offline = await readOfflineList(mailboxId ?? null, { + limit, + offset: position, + slot: await slotFor(client), + }); + if (!offline || offline.emails.length === 0) return online; + return { emails: offline.emails, hasMore: offline.hasMore, total: offline.total }; + }; + + const mailboxFallback = async ( + online: Mailbox[], + before: number, + accountId?: string, + ): Promise => { + // Bail out unless the result is EMPTY or is the exact synthetic placeholder. + // Testing `length > 1` here was a real bug found by + // `lib/__tests__/offline-fallback-client.test.ts`: a server that legitimately + // exposes a single mailbox got its real folder - real id, real counts - + // replaced by replica rows the moment any unrelated transport blip was + // recorded during the call. + if (online.length > 0 && !isSyntheticMailboxList(online)) return online; + if (isReplicaUnavailable()) return online; + if (transportFailureCount() === before) return online; + if (!scopedToPrimary(client, accountId)) return online; + const offline = await readOfflineMailboxes(await slotFor(client)); + if (!offline || offline.length === 0) return online; + return offline; + }; + + target.getMailboxes = async (accountId?: string): Promise => { + const before = transportFailureCount(); + const online = await originalGetMailboxes(accountId); + return mailboxFallback(online, before, accountId); + }; + + target.getAllMailboxes = async (): Promise => { + const before = transportFailureCount(); + const online = await originalGetAllMailboxes(); + // `getAllMailboxes` falls back internally to `getMailboxes()`, so an offline + // run arrives here as the synthetic single Inbox rather than an empty list. + return mailboxFallback(online, before); + }; + + return client; +} diff --git a/lib/offline-replica-client.ts b/lib/offline-replica-client.ts new file mode 100644 index 00000000..ba32c8b8 --- /dev/null +++ b/lib/offline-replica-client.ts @@ -0,0 +1,271 @@ +// Renderer-side client for the offline mail replica. +// +// The replica is EVENT-DRIVEN, exactly like the search index next to it: the +// renderer already holds the live JMAP push connection, so a `StateChange` is what +// triggers a sync cycle. There is no polling loop and no background worker. +// +// One cycle is BOUNDED (see lib/offline-replica/sync.ts's BUDGET), so a first +// sync of a large mailbox needs several. `unfinishedWork` is the server saying +// "call again", and `chainSync` below does that with a hard cap - the cap matters, +// because an "unfinished work" signal that is true for a condition the cycle +// cannot change is how the mobile client ended up chaining a new cycle every five +// seconds forever. +// +// Every function here is best-effort and never throws: offline storage failing to +// update must never break the mail UI. + +import { apiFetch } from '@/lib/browser-navigation'; +import { debug } from '@/lib/debug'; +import type { Email, Mailbox } from '@/lib/jmap/types'; +import type { StateChange } from '@/lib/jmap/types'; + +export interface RetentionPolicy { + envelopeDays: number; + bodyDays: number; + maxBodyMB: number; +} + +export interface CycleReport { + ok: boolean; + unfinishedWork: boolean; + bootstrapped: boolean; + reconciled: boolean; + mailboxesWritten: number; + envelopesWritten: number; + envelopesDeleted: number; + bodiesWritten: number; + bodiesEvicted: number; + coveragePhase: string; + resyncRequired: boolean; + warnings: string[]; + errorClass?: string; + error?: string; + durationMs: number; +} + +export interface ReplicaStats { + mailboxes: number; + envelopes: number; + bodies: number; + bodyBytes: number; + wantedBodies: number; + giveUps: number; + newest: string | null; + oldest: string | null; + fileBytes: number; +} + +export interface ReplicaStatus { + ok: boolean; + policy: RetentionPolicy; + limits: Record; + synced: boolean; + stats: ReplicaStats | null; + coveragePhase: string; + coveredFrom?: string | null; + resyncRequired: boolean; + lastCycleAt: number | null; + lastCycleOk: boolean | null; + lastCycleError?: string | null; +} + +/** Set once the server says the feature isn't there, so we stop asking. */ +let knownUnavailable = false; +let inFlight: Promise | null = null; + +function slotQuery(slot?: number, extra?: string): string { + const params = new URLSearchParams(); + if (typeof slot === 'number') params.set('slot', String(slot)); + const base = params.toString(); + if (extra && base) return `?${base}&${extra}`; + if (extra) return `?${extra}`; + return base ? `?${base}` : ''; +} + +/** True when the replica is known to be absent (not the desktop shell, or gated off). */ +export function isReplicaUnavailable(): boolean { + return knownUnavailable; +} + +export function resetReplicaAvailability(): void { + knownUnavailable = false; +} + +/** + * Runs ONE cycle. Single-flighted on the renderer as well as the server, so a + * burst of deliveries coalesces instead of queueing N overlapping requests that + * the server would then serialise anyway. + */ +export async function syncOnce( + opts: { slot?: number; policy?: RetentionPolicy; forceResync?: boolean } = {}, +): Promise { + if (knownUnavailable) return null; + if (inFlight) return inFlight; + + const run = (async (): Promise => { + try { + const response = await apiFetch(`/api/offline/sync${slotQuery(opts.slot)}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ policy: opts.policy, forceResync: opts.forceResync === true }), + }); + // 404 = not the desktop shell. Permanent for this page load; stop asking so + // a busy mailbox doesn't post per delivery. + if (response.status === 404) { knownUnavailable = true; return null; } + if (response.status === 503) { + const body = await response.json().catch(() => ({})); + // A transport-class 503 means the BACKEND is unreachable, which is normal + // and temporary - it must not latch the feature off for the session. Only + // a missing binding / key channel does that. + const code = typeof body?.code === 'string' ? body.code : ''; + if (code === 'no-binding' || code === 'no-key-channel' || code === 'unavailable') { + knownUnavailable = true; + return null; + } + return null; + } + if (!response.ok) return null; + const body = await response.json(); + debug.log('push', '[replica] cycle', body?.report); + return (body?.report ?? null) as CycleReport | null; + } catch { + return null; + } finally { + inFlight = null; + } + })(); + + inFlight = run; + return run; +} + +/** Hard cap on chained cycles per trigger. */ +export const MAX_CHAINED_CYCLES = 12; + +/** + * Runs cycles while the server reports unfinished work. + * + * The cap is the whole point. `unfinishedWork` is a hint, and a hint that stays + * true for something the cycle cannot resolve turns into an endless chain - which + * is exactly what happened on the mobile client when a body-queue counter reported + * attempted rather than inserted rows. The server-side fixes make that + * self-terminating; this cap means even a future regression costs a bounded number + * of requests rather than an infinite loop. + */ +export async function chainSync( + opts: { slot?: number; max?: number; onReport?: (report: CycleReport) => void } = {}, +): Promise { + const max = Math.min(opts.max ?? MAX_CHAINED_CYCLES, MAX_CHAINED_CYCLES); + let last: CycleReport | null = null; + for (let i = 0; i < max; i++) { + const report = await syncOnce({ slot: opts.slot }); + if (!report) return last; + last = report; + opts.onReport?.(report); + if (!report.ok || !report.unfinishedWork) return report; + } + return last; +} + +/** The push-driven entry point. Fire-and-forget: the mail UI must not wait on it. */ +export function syncOnStateChange(change: StateChange, opts: { slot?: number } = {}): void { + if (knownUnavailable) return; + // Only mail-shaped changes are worth a cycle. A `Mailbox` state change alone is + // usually just an unread-count move, but the replica DOES hold those counts, so + // unlike the search index it is worth reacting to. + const relevant = Object.values(change.changed ?? {}).some( + (perAccount) => perAccount && (perAccount.Email || perAccount.Mailbox), + ); + if (!relevant) return; + void syncOnce({ slot: opts.slot }); +} + +export async function fetchReplicaStatus(slot?: number): Promise { + try { + const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`); + if (!response.ok) return null; + return (await response.json()) as ReplicaStatus; + } catch { + return null; + } +} + +export async function updateRetentionPolicy( + policy: RetentionPolicy, + slot?: number, +): Promise { + try { + const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(policy), + }); + return response.ok; + } catch { + return false; + } +} + +export async function purgeReplica(slot?: number): Promise { + try { + const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`, { method: 'DELETE' }); + return response.ok; + } catch { + return false; + } +} + +// ── reads ─────────────────────────────────────────────────────────────────── + +interface ReadEnvelope { + ok?: boolean; + available?: boolean; + error?: string; + data?: T; +} + +async function read(query: string): Promise<(T & { available: boolean }) | null> { + if (knownUnavailable) return null; + try { + const response = await apiFetch(`/api/offline/mail${query}`); + if (response.status === 404) { knownUnavailable = true; return null; } + if (!response.ok) return null; + const body = (await response.json()) as ReadEnvelope & Record; + if (body?.available !== true) return null; + return body as unknown as T & { available: boolean }; + } catch { + return null; + } +} + +export async function readOfflineMailboxes(slot?: number): Promise { + const body = await read<{ mailboxes: Mailbox[] }>(slotQuery(slot, 'kind=mailboxes')); + return body?.mailboxes ?? null; +} + +export async function readOfflineList( + mailboxId: string | null, + opts: { limit?: number; offset?: number; slot?: number } = {}, +): Promise<{ emails: Email[]; total: number; hasMore: boolean } | null> { + const params = new URLSearchParams({ kind: 'list' }); + if (mailboxId !== null) params.set('mailboxId', mailboxId); + if (opts.limit !== undefined) params.set('limit', String(opts.limit)); + if (opts.offset !== undefined) params.set('offset', String(opts.offset)); + const body = await read<{ emails: Email[]; total: number; hasMore: boolean }>( + slotQuery(opts.slot, params.toString()), + ); + if (!body) return null; + return { emails: body.emails ?? [], total: body.total ?? 0, hasMore: body.hasMore === true }; +} + +export async function readOfflineMessage( + id: string, + slot?: number, +): Promise<{ email: Email | null; hasBody: boolean } | null> { + const params = new URLSearchParams({ kind: 'message', id }); + const body = await read<{ email: Email | null; hasBody: boolean }>( + slotQuery(slot, params.toString()), + ); + if (!body) return null; + return { email: body.email ?? null, hasBody: body.hasBody === true }; +} diff --git a/lib/offline-replica/__tests__/apply.test.ts b/lib/offline-replica/__tests__/apply.test.ts new file mode 100644 index 00000000..442a2776 --- /dev/null +++ b/lib/offline-replica/__tests__/apply.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from 'vitest'; +import { + advanceOneMs, madeForwardProgress, normalisePage, pageIsEmpty, planEmailFetches, + planMailboxFetches, updatedPropertiesAreCountsOnly, type ChangesPage, +} from '../apply'; +import { asChangesState } from '../states'; + +function page(partial: Partial): ChangesPage { + return { + oldState: asChangesState('old'), + newState: asChangesState('new'), + hasMoreChanges: false, + created: [], + updated: [], + destroyed: [], + ...partial, + }; +} + +describe('normalisePage', () => { + it('lets a destroyed id win outright over created and updated', () => { + // Fetching an id that is also destroyed spends a request to get `notFound`. + const out = normalisePage(page({ created: ['a', 'b'], updated: ['a'], destroyed: ['a'] })); + expect(out.created).toEqual(['b']); + expect(out.updated).toEqual([]); + expect(out.destroyed).toEqual(['a']); + }); + + it('treats an id in both created and updated as a create', () => { + // The create path fetches the full envelope tier, which already contains the + // updated values - so an extra 3-property fetch would be pure waste. + const out = normalisePage(page({ created: ['a'], updated: ['a'] })); + expect(out.created).toEqual(['a']); + expect(out.updated).toEqual([]); + }); + + it('deduplicates within each bucket', () => { + const out = normalisePage(page({ created: ['a', 'a'], destroyed: ['b', 'b'] })); + expect(out.created).toEqual(['a']); + expect(out.destroyed).toEqual(['b']); + }); +}); + +describe('pageIsEmpty', () => { + it('is true only when nothing changed', () => { + // An empty page STILL has to advance the cursor: skipping it re-requests the + // same position forever. + expect(pageIsEmpty(page({}))).toBe(true); + expect(pageIsEmpty(page({ updated: ['a'] }))).toBe(false); + }); +}); + +describe('planEmailFetches', () => { + it('drops an updated id we do not hold locally, BEFORE any fetch is issued', () => { + // The absent case is an unconditional no-op. Fetching it would need a + // `receivedAt` the 3-property response cannot supply and the schema's + // NOT NULL would reject. Coverage enumerates CURRENT state, so it will pick + // the record up with the updated values anyway. + const plan = planEmailFetches(page({ updated: ['have', 'missing'] }), new Set(['have'])); + expect(plan.updateIds).toEqual(['have']); + }); + + it('keeps creates unconditional - presence is irrelevant for a create', () => { + const plan = planEmailFetches(page({ created: ['new'] }), new Set()); + expect(plan.createIds).toEqual(['new']); + }); + + it('never routes an id into both the create and the update fetch', () => { + const plan = planEmailFetches(page({ created: ['a'], updated: ['a'] }), new Set(['a'])); + expect(plan.createIds).toEqual(['a']); + expect(plan.updateIds).toEqual([]); + }); +}); + +describe('updatedPropertiesAreCountsOnly', () => { + it('is true for the four counters and for an empty list', () => { + expect(updatedPropertiesAreCountsOnly(['unreadEmails'])).toBe(true); + expect(updatedPropertiesAreCountsOnly(['totalEmails', 'unreadThreads'])).toBe(true); + // "nothing but the state token moved" is counts-only vacuously. + expect(updatedPropertiesAreCountsOnly([])).toBe(true); + }); + + it('is false when the server will not say what changed', () => { + // `null` means "assume everything", so the whole object must be re-fetched. + expect(updatedPropertiesAreCountsOnly(null)).toBe(false); + expect(updatedPropertiesAreCountsOnly(undefined)).toBe(false); + }); + + it('is false as soon as one non-count property is present', () => { + expect(updatedPropertiesAreCountsOnly(['unreadEmails', 'name'])).toBe(false); + }); +}); + +describe('planMailboxFetches', () => { + it('routes updates to the cheap four-integer patch when only counts moved', () => { + const plan = planMailboxFetches( + page({ created: ['new'], updated: ['old'], updatedProperties: ['unreadEmails'] }), + ); + expect(plan.fullIds).toEqual(['new']); + expect(plan.countOnlyIds).toEqual(['old']); + }); + + it('re-fetches the whole object when updatedProperties is null', () => { + const plan = planMailboxFetches(page({ updated: ['old'], updatedProperties: null })); + expect(plan.fullIds).toEqual(['old']); + expect(plan.countOnlyIds).toEqual([]); + }); +}); + +describe('keyset progress', () => { + it('requires STRICTLY greater, because `after` is spec-inclusive', () => { + // RFC 8621 s4.4.1: receivedAt "must be the same or after this date-time to + // match". So every page re-returns the boundary message, and equality is NOT + // progress - treating it as progress would loop on that millisecond forever. + expect(madeForwardProgress('2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z')).toBe(false); + expect(madeForwardProgress('2026-01-01T00:00:00.001Z', '2026-01-01T00:00:00.000Z')).toBe(true); + expect(madeForwardProgress(null, '2026-01-01T00:00:00.000Z')).toBe(false); + expect(madeForwardProgress('2026-01-01T00:00:00.000Z', null)).toBe(true); + }); + + it('advances exactly one millisecond in the last-resort rung', () => { + expect(advanceOneMs('2026-01-01T00:00:00.000Z')).toBe('2026-01-01T00:00:00.001Z'); + // A malformed value must not become NaN and poison the cursor. + expect(advanceOneMs('not-a-date')).toBe('not-a-date'); + }); +}); diff --git a/lib/offline-replica/__tests__/errors.test.ts b/lib/offline-replica/__tests__/errors.test.ts new file mode 100644 index 00000000..89033ccf --- /dev/null +++ b/lib/offline-replica/__tests__/errors.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest'; +import { + backoffDelayMs, classify, escalationApplies, movesCursor, nextRung, rungValue, + type ErrorClass, +} from '../errors'; + +const ALL: ErrorClass[] = [ + 'Transport', 'RateLimit', 'ServerTransient', 'RequestLimit', 'Auth', 'Fatal', 'StateInvalid', +]; + +describe('exactly one class moves a cursor', () => { + it('is StateInvalid, and nothing else', () => { + // This is the single load-bearing property of the taxonomy. Every other class + // leaves the cursor exactly where it was, which is what makes "a failure never + // causes silent data loss" structural rather than aspirational. + expect(ALL.filter(movesCursor)).toEqual(['StateInvalid']); + }); + + it('escalates to a rebuild only for size/availability problems', () => { + // Escalating on RateLimit would answer a rate-limited server with far MORE + // requests. On Auth, a 401 would trigger a rebuild. On Transport, a flaky + // tunnel would. Fatal is our own bug and a rebuild will not fix it. + expect(ALL.filter(escalationApplies).sort()).toEqual(['RequestLimit', 'ServerTransient']); + }); +}); + +describe('classify', () => { + it('reads HTTP status before anything else', () => { + expect(classify({ httpStatus: 401 })).toBe('Auth'); + expect(classify({ httpStatus: 403 })).toBe('Auth'); + expect(classify({ httpStatus: 429 })).toBe('RateLimit'); + expect(classify({ httpStatus: 413 })).toBe('RequestLimit'); + expect(classify({ httpStatus: 503 })).toBe('ServerTransient'); + }); + + it('classifies cannotCalculateChanges as the one cursor-moving class', () => { + expect(classify({ jmapErrorType: 'cannotCalculateChanges' })).toBe('StateInvalid'); + }); + + it('defaults an UNRECOGNISED method error to ServerTransient', () => { + // Guessing transient costs a retry; guessing state-invalid costs a full + // resync; guessing fatal stalls the account. The cheapest wrong answer wins. + expect(classify({ jmapErrorType: 'somethingNobodyHasHeardOf' })).toBe('ServerTransient'); + }); + + it('does not let a method error description masquerade as a transport failure', () => { + // Structure before strings: a method error's prose can legitimately contain + // "timeout" or "socket", and reading that as Transport would leave a genuine + // server-side problem being retried as though the network were down. + expect(classify({ jmapErrorType: 'invalidArguments', message: 'socket timeout' })).toBe('Fatal'); + }); + + it('classifies a real fetch rejection as Transport', () => { + // "Offline is not an error": the cursor stands still and the work is retried. + for (const message of [ + 'fetch failed', 'connect ECONNREFUSED 127.0.0.1:1', 'getaddrinfo ENOTFOUND nope', + 'socket hang up', 'The operation timed out', + ]) { + expect(classify({ message }), message).toBe('Transport'); + } + }); +}); + +describe('the maxChanges ladder is monotonically non-increasing for EVERY server value', () => { + it('never proposes a retry larger than the attempt that just failed', () => { + // Two historical bugs live here. An unbounded middle rung produced a retry + // STRICTLY LARGER than the failing attempt, actively worsening a + // "response too large" error. Clamping only rung 0 then reintroduced it in a + // narrower form: maxObjectsInGet=100 gave rung0=100 and rung1=250. + const serverValues = [ + undefined, 1, 5, 10, 20, 25, 26, 49, 50, 51, 99, 100, 249, 250, 251, 499, 500, 501, 5000, + ]; + for (const value of serverValues) { + const rungs = ([0, 1, 2, 3] as const).map((r) => rungValue(r, value)); + for (let i = 1; i < rungs.length; i++) { + expect( + rungs[i], + `maxObjectsInGet=${value} rung ${i} (${rungs[i]}) must not exceed rung ${i - 1} (${rungs[i - 1]})`, + ).toBeLessThanOrEqual(rungs[i - 1]); + } + // And never zero, or the request asks for nothing and never progresses. + for (const r of rungs) expect(r).toBeGreaterThanOrEqual(1); + } + }); + + it('clamps rung 0 to what the server allows', () => { + expect(rungValue(0, 100)).toBe(100); + expect(rungValue(0, 5000)).toBe(500); + expect(rungValue(0, undefined)).toBe(500); + }); + + it('saturates rather than running off the end of the ladder', () => { + expect(nextRung(0)).toBe(1); + expect(nextRung(3)).toBe(3); + }); +}); + +describe('backoff', () => { + it('is full-jitter and bounded by the cap', () => { + for (let attempt = 0; attempt < 12; attempt++) { + const delay = backoffDelayMs(attempt, { baseMs: 1000, capMs: 60_000 }); + expect(delay).toBeGreaterThanOrEqual(0); + expect(delay).toBeLessThanOrEqual(60_000); + } + }); +}); diff --git a/lib/offline-replica/__tests__/retention.test.ts b/lib/offline-replica/__tests__/retention.test.ts new file mode 100644 index 00000000..4f8377f5 --- /dev/null +++ b/lib/offline-replica/__tests__/retention.test.ts @@ -0,0 +1,154 @@ +// The clock-jump guard, and the wipe it caused on the mobile client. +// +// The bug being regressed here is not hypothetical: its reproduction on the mobile +// side returned 0 envelopes from a 4-envelope store. The guard DETECTED the jump, +// held the old floor for one cycle - and persisted the JUMPED floor. The next +// chained cycle seconds later computed a floor within seconds of the persisted +// one, so the guard passed, the movement was classified as a NARROW, and every +// envelope below a floor a year in the future was evicted. Unrecoverable, because +// `coveredFrom` then claims the range complete and `/changes` cannot re-deliver +// pre-existing mail. + +import { describe, expect, it } from 'vitest'; +import { + adjustForWindow, CLOCK_JUMP_GUARD_MS, computeFloors, floorMovement, + guardFloorAgainstClockJump, +} from '../retention'; + +const DAY = 24 * 60 * 60 * 1000; +const T0 = Date.parse('2026-08-05T12:00:00.000Z'); + +function iso(t: number): string { + return new Date(t).toISOString(); +} + +describe('computeFloors', () => { + it('never lets the body window be wider than the envelope window', () => { + // A body with no envelope is an orphan by construction, and the whole point of + // two tiers is envelopes being a superset of bodies. + const floors = computeFloors({ envelopeDays: 30, bodyDays: 365, maxBodyMB: 100 }, T0); + expect(floors.bodyFrom).toBe(floors.envelopeFrom); + }); + + it('turns the MB cap into bytes', () => { + expect(computeFloors({ envelopeDays: 1, bodyDays: 1, maxBodyMB: 2 }, T0).maxBodyBytes) + .toBe(2 * 1024 * 1024); + }); +}); + +describe('guardFloorAgainstClockJump', () => { + it('adopts the computed floor when there is no history to compare against', () => { + const g = guardFloorAgainstClockJump(iso(T0), undefined); + expect(g.suppressed).toBe(false); + expect(g.evictionAllowed).toBe(true); + expect(g.envelopeFrom).toBe(iso(T0)); + }); + + it('adopts an ordinary drift - a DST shift must not trip it', () => { + const g = guardFloorAgainstClockJump(iso(T0 + 60 * 60 * 1000), iso(T0)); + expect(g.suppressed).toBe(false); + expect(g.evictionAllowed).toBe(true); + }); + + it('suppresses a jump larger than the guard and refuses to authorise deletion', () => { + const jumped = iso(T0 + 365 * DAY); + const g = guardFloorAgainstClockJump(jumped, iso(T0)); + expect(g.suppressed).toBe(true); + expect(g.envelopeFrom).toBe(iso(T0)); + // Suppressing the FLOOR is not the same as suppressing the DELETIONS the + // floor authorises. Both the retention eviction and the reconcile sweep read + // this bit. + expect(g.evictionAllowed).toBe(false); + expect(g.warning).toBeTruthy(); + }); + + it('THE H2 REGRESSION: persists the floor it USED, not the one it rejected', () => { + // This one assertion is the whole fix. Persisting the computed value here is + // what legitimised the anomaly on the very next cycle. + const jumped = iso(T0 + 365 * DAY); + const g = guardFloorAgainstClockJump(jumped, iso(T0)); + expect(g.nextLastWindowFloor).toBe(iso(T0)); + expect(g.nextLastWindowFloor).not.toBe(jumped); + }); + + it('THE H2 REGRESSION: stays suppressed across MANY chained cycles', () => { + // The original bug only showed on the SECOND cycle, so a single-cycle test + // passes against the broken code. Chaining is what reproduces it. + const stored = iso(T0); + let lastWindowFloor: string | undefined = stored; + for (let cycle = 0; cycle < 20; cycle++) { + // The clock is a year ahead and creeping forward a few seconds per cycle, + // exactly as a chained sync would observe it. + const computed = iso(T0 + 365 * DAY + cycle * 5_000); + const g = guardFloorAgainstClockJump(computed, lastWindowFloor); + expect(g.suppressed, `cycle ${cycle} must stay suppressed`).toBe(true); + expect(g.evictionAllowed, `cycle ${cycle} must not authorise deletion`).toBe(false); + expect(g.envelopeFrom, `cycle ${cycle} must keep the original floor`).toBe(stored); + lastWindowFloor = g.nextLastWindowFloor; + } + // And after 20 cycles the remembered floor is still the trustworthy one, so + // no later cycle can classify it as a narrow and evict everything. + expect(lastWindowFloor).toBe(stored); + expect(floorMovement(lastWindowFloor, stored)).toBe('unchanged'); + expect(adjustForWindow(floorMovement(lastWindowFloor, stored), stored).evictBelow) + .toBeUndefined(); + }); + + it('suppresses a BACKWARD jump too', () => { + const g = guardFloorAgainstClockJump(iso(T0 - 365 * DAY), iso(T0)); + expect(g.suppressed).toBe(true); + expect(g.evictionAllowed).toBe(false); + }); + + it('treats an explicit retention change as INTENT and applies it, eviction included', () => { + // The computed floor moves for two independent reasons - the clock changing + // and the SETTING changing - and guarding a setting change is wrong. Without + // this discriminator a Settings edit sits unapplied until something unrelated + // moves the floor again. + const widened = iso(T0 - 365 * DAY); + const g = guardFloorAgainstClockJump(widened, iso(T0), { policyChanged: true }); + expect(g.suppressed).toBe(false); + expect(g.evictionAllowed).toBe(true); + expect(g.envelopeFrom).toBe(widened); + expect(g.nextLastWindowFloor).toBe(widened); + }); + + it('a genuine user NARROW still evicts', () => { + // The guard must not become a reason nothing is ever deleted. + const narrowed = iso(T0 + 20 * 60 * 60 * 1000); + const g = guardFloorAgainstClockJump(narrowed, iso(T0)); + expect(g.evictionAllowed).toBe(true); + expect(floorMovement(iso(T0), g.envelopeFrom)).toBe('narrowed'); + expect(adjustForWindow('narrowed', g.envelopeFrom).evictBelow).toBe(narrowed); + }); + + it('tolerates an unparseable stored floor without wedging', () => { + const g = guardFloorAgainstClockJump(iso(T0), 'garbage'); + // Date.parse('garbage') is NaN, so the delta is not finite: adopt rather than + // suppress forever on a corrupt value. + expect(g.suppressed).toBe(false); + }); + + it('uses a threshold above a day so a leap second or NTP nudge is invisible', () => { + expect(CLOCK_JUMP_GUARD_MS).toBeGreaterThan(DAY); + }); +}); + +describe('floorMovement / adjustForWindow', () => { + it('a LATER floor keeps less mail and means evict', () => { + expect(floorMovement(iso(T0), iso(T0 + DAY))).toBe('narrowed'); + expect(adjustForWindow('narrowed', iso(T0 + DAY))).toEqual({ evictBelow: iso(T0 + DAY) }); + }); + + it('an EARLIER floor means re-scan, NOT a resync', () => { + // A widen moves the target back and re-enters coverage scanning. The cursors + // are untouched - a widen is not a reason to rebuild. + expect(floorMovement(iso(T0), iso(T0 - DAY))).toBe('widened'); + expect(adjustForWindow('widened', iso(T0 - DAY))).toEqual({ rescanFrom: iso(T0 - DAY) }); + }); + + it('does nothing without a previous floor', () => { + expect(floorMovement(undefined, iso(T0))).toBe('unchanged'); + expect(adjustForWindow('unchanged', iso(T0))).toEqual({}); + }); +}); diff --git a/lib/offline-replica/__tests__/states.test.ts b/lib/offline-replica/__tests__/states.test.ts new file mode 100644 index 00000000..2d372345 --- /dev/null +++ b/lib/offline-replica/__tests__/states.test.ts @@ -0,0 +1,99 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + asChangesState, asSnapshotState, coveragePhaseForCommitment, mintEnumerationCommitment, +} from '../states'; + +describe('state token certification', () => { + it('rejects everything a parsed JSON body could hand over that is not a token', () => { + // The brand certifies PROVENANCE; this check certifies SHAPE. Without it a + // `null` or a number could be laundered into something the engine treats as a + // cursor forever. + for (const bad of [null, undefined, 0, 1, '', {}, [], true]) { + expect(() => asChangesState(bad)).toThrow(TypeError); + expect(() => asSnapshotState(bad)).toThrow(TypeError); + } + }); + + it('accepts a non-empty string', () => { + expect(asChangesState('s1')).toBe('s1'); + expect(asSnapshotState('s1')).toBe('s1'); + }); +}); + +describe('EnumerationCommitment', () => { + it('is constructible - the symbol tag must be a real runtime Symbol', () => { + // `declare const tag: unique symbol` is type-level only and emits no runtime + // value, so using it as a computed key throws ReferenceError the first time + // the mint runs. That mistake is in the superseded design document; this test + // is what catches it. + const commitment = mintEnumerationCommitment({ + jmapAccountId: 'a', + snapshot: asSnapshotState('snap'), + targetFrom: '2026-01-01T00:00:00.000Z', + sweepFloor: '2026-01-01T00:00:00.000Z', + kind: 'bootstrap', + }); + expect(commitment.snapshot).toBe('snap'); + expect(commitment.kind).toBe('bootstrap'); + }); + + it('maps its kind onto the coverage phase', () => { + const base = { + jmapAccountId: 'a', + snapshot: asSnapshotState('snap'), + targetFrom: 'x', + sweepFloor: 'x', + } as const; + expect(coveragePhaseForCommitment(mintEnumerationCommitment({ ...base, kind: 'bootstrap' }))) + .toBe('scanning'); + expect(coveragePhaseForCommitment(mintEnumerationCommitment({ ...base, kind: 'reconcile' }))) + .toBe('reconciling'); + }); + + it('does not export its tag, so no object literal elsewhere can forge the type', () => { + const source = fs.readFileSync(path.join(__dirname, '..', 'states.ts'), 'utf8'); + expect(source).toContain("const enumerationCommitmentTag = Symbol('EnumerationCommitment')"); + expect(source).not.toMatch(/export\s+(const|let)\s+enumerationCommitmentTag/); + // And it must be a real Symbol() call, not the type-only declaration form. + expect(source).not.toMatch(/declare\s+const\s+enumerationCommitmentTag/); + }); +}); + +describe('cursor provenance is greppable, not just documented', () => { + const replicaDir = path.join(__dirname, '..'); + + function sourceFiles(): string[] { + return fs + .readdirSync(replicaDir) + .filter((f) => f.endsWith('.ts')) + .map((f) => path.join(replicaDir, f)); + } + + it('mints branded states ONLY in jmap.ts (the response parser)', () => { + // This is the rule the whole brand exists to enforce. The mobile client's + // defect D4 was a snapshot state adopted as a /changes cursor after a + // transient 503; a cast anywhere outside the parser is how that comes back. + for (const file of sourceFiles()) { + const base = path.basename(file); + if (base === 'states.ts' || base === 'jmap.ts') continue; + const source = fs.readFileSync(file, 'utf8'); + expect(source, `${base} must not mint a ChangesState`).not.toMatch(/asChangesState\s*\(/); + expect(source, `${base} must not mint a SnapshotState`).not.toMatch(/asSnapshotState\s*\(/); + expect(source, `${base} must not cast to a branded state`).not.toMatch( + /as\s+(ChangesState|SnapshotState)\b/, + ); + } + }); + + it('mints an EnumerationCommitment ONLY where an enumeration is actually started', () => { + // A commitment is a promise to enumerate. Minting one anywhere that does not + // then enumerate makes the seed path's teeth meaningless. + const callers = sourceFiles().filter((file) => { + if (path.basename(file) === 'states.ts') return false; + return /mintEnumerationCommitment\s*\(/.test(fs.readFileSync(file, 'utf8')); + }); + expect(callers.map((f) => path.basename(f))).toEqual(['sync.ts']); + }); +}); diff --git a/lib/offline-replica/__tests__/store.test.ts b/lib/offline-replica/__tests__/store.test.ts new file mode 100644 index 00000000..0bf20bc6 --- /dev/null +++ b/lib/offline-replica/__tests__/store.test.ts @@ -0,0 +1,502 @@ +// Store-level invariants, against a REAL SQLCipher file. +// +// Skipped wholesale when the optional native binding is not installed (that is a +// normal state on a platform with no prebuild - see lib/mail-index/binding.ts), so +// this file must never be the only proof of anything. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { randomBytes } from 'node:crypto'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { isSqlcipherAvailable } from '@/lib/mail-index/binding'; +import { indexDbPath } from '@/lib/mail-index/paths'; +import { clampPolicy, DEFAULT_POLICY, ReplicaStore } from '../store'; +import { reconcileStamp } from '../sync'; +import { asChangesState, asSnapshotState, mintEnumerationCommitment } from '../states'; +import type { EnvelopeRow } from '../types'; + +const ACCOUNT = 'alice@example.org'; +const JMAP = 'jmap-account-1'; + +function envelope(id: string, receivedAt: string, extra: Partial = {}): EnvelopeRow { + return { + jmapAccountId: JMAP, + id, + threadId: `t-${id}`, + receivedAt, + size: 1000, + subject: `subject ${id}`, + preview: `preview ${id}`, + fromJson: JSON.stringify([{ email: 'sender@example.org' }]), + toJson: null, + ccJson: null, + blobId: `blob-${id}`, + hasAttachment: false, + keywordsJson: '{}', + mailboxIds: ['inbox'], + ...extra, + }; +} + +describe.skipIf(!isSqlcipherAvailable())('ReplicaStore', () => { + let storeDir: string; + let key: Buffer; + let store: ReplicaStore; + + beforeEach(() => { + storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-test-')); + key = randomBytes(32); + store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key }); + }); + + afterEach(() => { + store.close(); + fs.rmSync(storeDir, { recursive: true, force: true }); + }); + + it('writes into the SAME file as the search index, and it is really encrypted', () => { + // One encryption boundary, one key, one purge. And `PRAGMA key` is a silent + // no-op on a non-SQLCipher binding, so the header check is the only thing that + // catches a store that "works" while sitting on disk in cleartext. + expect(store.dbPath).toBe(indexDbPath(storeDir, ACCOUNT)); + store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); }); + store.close(); + const header = fs.readFileSync(store.dbPath).subarray(0, 15).toString('latin1'); + expect(header).not.toBe('SQLite format 3'); + const raw = Buffer.concat( + ['', '-wal', '-shm'] + .map((s) => `${store.dbPath}${s}`) + .filter((f) => fs.existsSync(f)) + .map((f) => fs.readFileSync(f)), + ); + expect(raw.includes('subject e1')).toBe(false); + // Re-open so afterEach's close() is harmless. + store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key }); + }); + + describe('cursor provenance at the storage layer', () => { + it('refuses to create a cursor from nowhere', () => { + // A cursor is born from seedCursor and nowhere else. Creating one in + // advanceCursor would be a silent cursor-from-nowhere - exactly what the + // branded types exist to make impossible. + expect(() => store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s1'))) + .toThrow(/seed it first/); + }); + + it('writes the cursor AND the coverage row it justifies in one transaction', () => { + store.transaction(() => { + store.seedCursor( + { jmapAccountId: JMAP, type: 'Email' }, + mintEnumerationCommitment({ + jmapAccountId: JMAP, + snapshot: asSnapshotState('snap-1'), + targetFrom: '2026-01-01T00:00:00.000Z', + sweepFloor: '2026-01-01T00:00:00.000Z', + kind: 'bootstrap', + }), + 1000, + ); + }); + expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('snap-1'); + const coverage = store.getCoverage(JMAP); + expect(coverage?.phase).toBe('scanning'); + expect(coverage?.sweepFloor).toBe('2026-01-01T00:00:00.000Z'); + }); + + it('rolls back a seed whose commitment is for the wrong account', () => { + expect(() => + store.transaction(() => { + store.seedCursor( + { jmapAccountId: JMAP, type: 'Email' }, + mintEnumerationCommitment({ + jmapAccountId: 'someone-else', + snapshot: asSnapshotState('snap'), + targetFrom: 'x', sweepFloor: 'x', kind: 'bootstrap', + }), + 1000, + ); + }), + ).toThrow(/different JMAP account/); + expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull(); + }); + + it('advances a seeded cursor and keeps counters field-level', () => { + seed(store); + store.transaction(() => { + store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s2')); + store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { consecutiveFailures: 3 }); + }); + const cursor = store.getCursor({ jmapAccountId: JMAP, type: 'Email' }); + expect(cursor?.state).toBe('s2'); + expect(cursor?.consecutiveFailures).toBe(3); + // A patch must not be able to rewrite `state` - only advance/seed can. + store.transaction(() => { + store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { drainPending: true }); + }); + expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('s2'); + }); + }); + + describe('envelope tier', () => { + it('does NOT reset has_body on an idempotent replay', () => { + // Otherwise a replayed page looks like "body missing" to the backfill job and + // re-downloads every body in the page. + store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); }); + store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{}}'); }); + expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10)).toHaveLength(0); + store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 2); }); + expect( + store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10), + 'a replayed envelope upsert must not clear has_body', + ).toHaveLength(0); + }); + + it('patches only the two mutable properties, and no-ops for an absent id', () => { + store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); }); + const ok = store.transaction(() => + store.patchEnvelopeMutable(JMAP, 'e1', { keywordsJson: '{"$seen":true}', mailboxIds: ['archive'] }), + ); + expect(ok).toBe(true); + expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual(['archive']); + // An update for an id we do not hold must leave no membership rows behind. + const missing = store.transaction(() => + store.patchEnvelopeMutable(JMAP, 'nope', { keywordsJson: '{}', mailboxIds: ['inbox'] }), + ); + expect(missing).toBe(false); + expect(store.mailboxIdsFor(JMAP, 'nope')).toEqual([]); + }); + + it('never writes a body whose envelope is gone', () => { + // A body fetched moments before its envelope was destroyed in the same cycle + // would otherwise land as an orphan. + const wrote = store.transaction(() => store.putBodyIfEnvelopeExists(JMAP, 'ghost', '{}')); + expect(wrote).toBe(false); + expect(store.getBody(JMAP, 'ghost')).toBeNull(); + }); + + it('deleting an email takes its body, membership and queue row with it', () => { + store.transaction(() => { + store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); + store.enqueueBodies([{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }]); + }); + store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"a":1}'); }); + store.transaction(() => { store.deleteEmails(JMAP, ['e1']); }); + expect(store.getBody(JMAP, 'e1')).toBeNull(); + expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual([]); + expect(store.countWantedBodies(JMAP, Date.now())).toBe(0); + }); + }); + + describe('the reconcile sweep', () => { + it('refuses to run without a pinned stamp rather than deleting unverified records', () => { + expect(() => store.sweep(JMAP, '2026-01-01T00:00:00.000Z', undefined)) + .toThrow(/refusing to delete unverified/); + }); + + it('keeps what the enumeration re-saw and deletes what it did not', () => { + // The whole "seen set as one integer" trick: re-upserting refreshes + // cached_at, and the sweep deletes anything still below the pin. + store.transaction(() => { + store.upsertEnvelopes([ + envelope('kept', '2026-08-01T00:00:00.000Z'), + envelope('gone', '2026-08-02T00:00:00.000Z'), + ], 100); + }); + const stamp = Math.max(500, store.maxEnvelopeCachedAt(JMAP) + 1); + store.transaction(() => { + store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp); + }); + store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); }); + expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull(); + expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull(); + }); + + it('a stamp taken from a FROZEN clock would sweep nothing; the derived one works', () => { + // Both halves matter, and both fail silently. Exercising the real + // `reconcileStamp` rather than re-deriving it in the test is the point. + store.transaction(() => { + store.upsertEnvelopes([ + envelope('kept', '2026-08-01T00:00:00.000Z'), + envelope('gone', '2026-08-02T00:00:00.000Z'), + ], 9_999); + }); + const frozenNow = 1_000; + + // The naive version: with the clock behind the data, nothing is below the + // stamp, so a re-verified store sweeps zero rows and stale records live on. + expect(store.sweep(JMAP, '2026-07-01T00:00:00.000Z', frozenNow)).toBe(0); + + const stamp = reconcileStamp(frozenNow, store.maxEnvelopeCachedAt(JMAP)); + expect(stamp).toBe(10_000); + store.transaction(() => { + store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp); + }); + expect(store.transaction(() => store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp))).toBe(1); + expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull(); + expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull(); + }); + + it('stamping an enumeration with `now` instead of the pin deletes what it just verified', () => { + // The other direction of the same bug: the pin EXCEEDS now, so a page that + // stamps with `now` lands below the pin and the sweep eats it. + store.transaction(() => { + store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], 9_999); + }); + const now = 1_000; + const stamp = reconcileStamp(now, store.maxEnvelopeCachedAt(JMAP)); + // Re-verified against the server, but stamped with the WRONG value. + store.transaction(() => { + store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], now); + }); + store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); }); + expect( + store.getEnvelopeRaw(JMAP, 'verified'), + 'this is the failure mode the pinned stamp exists to prevent', + ).toBeNull(); + }); + }); + + describe('the body queue - the durable-terminal-state fixes', () => { + it('enqueueBodies reports rows ACTUALLY INSERTED, not attempted', () => { + // Reporting the attempted count made the mobile engine believe there was + // unfinished work every cycle for as long as any envelope lacked a body, + // chaining a new cycle every few seconds indefinitely. + const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }; + expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(1); + expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(0); + }); + + it('never resets attempts on a re-enqueue', () => { + const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }; + store.transaction(() => { store.enqueueBodies([entry]); }); + store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 0, 'boom'); }); + store.transaction(() => { store.enqueueBodies([entry]); }); + expect(store.takeBodyQueue(JMAP, 10, Date.now())[0]?.attempts).toBe(1); + }); + + it('THE H1 REGRESSION: a gave-up row is KEPT and is never revived by a re-enqueue', () => { + // Deleting the row on give-up was not enough: the backfill driver is + // "envelope with no body", which cannot tell "not fetched yet" from + // "deliberately not kept", so the next pass re-inserted a fresh attempts=0 + // row and a permanently-failing body was retried five times per cycle forever. + store.transaction(() => { + store.markBodyGaveUp(JMAP, [ + { emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' }, + ]); + }); + expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['e1']); + // Not WANTED any more, so the drain never picks it up again. + expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0); + // And a re-enqueue cannot resurrect it. + const inserted = store.transaction(() => + store.enqueueBodies([ + { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }, + ]), + ); + expect(inserted).toBe(0); + expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0); + }); + + it('THE H1c REGRESSION: a cap-shed body is markable even with NO existing queue row', () => { + // The download/discard loop: the cap sheds a body that was fetched and stored + // successfully, so there is no queue row left to UPDATE. If the mark is + // silently dropped, the envelope is still inside the body WINDOW, the backfill + // re-enqueues it, it downloads again, and the cap sheds it again - unbounded + // data use that never terminates. + store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); }); + store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{"1":{"value":"x"}}}'); }); + expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0); // no queue row exists + + store.transaction(() => { + store.deleteBodies(JMAP, ['e1']); + store.markBodyGaveUp(JMAP, [ + { emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' }, + ]); + }); + expect( + store.listBodyGiveUps(JMAP, 10), + 'the cap-shed mark must be an upsert, or the shed/re-download loop stays open', + ).toEqual(['e1']); + // The envelope is back to has_body=0 and still in the window, so without the + // mark the backfill WOULD pick it up. With the mark it is excluded. + expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10).map((e) => e.id)) + .toEqual(['e1']); + expect(store.listBodyGiveUps(JMAP, 10)).toContain('e1'); + }); + + it('clearing give-ups DELETES them, so they look like "never queued"', () => { + // A cleared give-up must come back with a clean attempt count, which an + // un-flag would not give. + store.transaction(() => { + store.markBodyGaveUp(JMAP, [ + { emailId: 'a', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' }, + { emailId: 'b', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' }, + ]); + }); + store.transaction(() => { store.clearBodyGiveUps(JMAP, 'shed-by-cap'); }); + expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['a']); + store.transaction(() => { store.clearBodyGiveUps(JMAP); }); + expect(store.listBodyGiveUps(JMAP, 10)).toEqual([]); + expect( + store.transaction(() => + store.enqueueBodies([ + { emailId: 'a', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }, + ]), + ), + 'a cleared give-up must be re-enqueueable', + ).toBe(1); + }); + + it('honours a backoff window', () => { + store.transaction(() => { + store.enqueueBodies([ + { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }, + ]); + }); + store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 10_000, 'later'); }); + expect(store.takeBodyQueue(JMAP, 10, 5_000)).toHaveLength(0); + expect(store.takeBodyQueue(JMAP, 10, 20_000)).toHaveLength(1); + }); + }); + + describe('eviction', () => { + it('cap eviction takes the oldest bodies first and leaves envelopes alone', () => { + store.transaction(() => { + store.upsertEnvelopes([ + envelope('old', '2026-01-01T00:00:00.000Z'), + envelope('new', '2026-08-01T00:00:00.000Z'), + ], 1); + }); + store.transaction(() => { + store.putBodyIfEnvelopeExists(JMAP, 'old', '{"v":"old"}'); + store.putBodyIfEnvelopeExists(JMAP, 'new', '{"v":"new"}'); + }); + expect(store.oldestBodies(JMAP, 1).map((b) => b.emailId)).toEqual(['old']); + store.transaction(() => { store.deleteBodies(JMAP, ['old']); }); + // The message stays LISTED - only its content went. + expect(store.getEnvelopeRaw(JMAP, 'old')).not.toBeNull(); + expect(store.countBodies(JMAP)).toBe(1); + }); + + it('no deletion path leaves an orphan body behind', () => { + // This is the real invariant. `orphanBodies()` is a belt-and-braces sweep for + // orphans a CRASH between two transactions could leave; it is deliberately + // not reachable through the store's own API, which is what this asserts. + // (So the detection query itself is covered only by the integration run, not + // by this file - stated rather than papered over with a vacuous assertion.) + store.transaction(() => { + store.upsertEnvelopes([ + envelope('a', '2026-01-01T00:00:00.000Z'), + envelope('b', '2026-08-01T00:00:00.000Z'), + ], 1); + }); + store.transaction(() => { + store.putBodyIfEnvelopeExists(JMAP, 'a', '{"v":1}'); + store.putBodyIfEnvelopeExists(JMAP, 'b', '{"v":2}'); + }); + expect(store.countBodies(JMAP)).toBe(2); + + store.transaction(() => { store.deleteEmails(JMAP, ['a']); }); + expect(store.orphanBodies(JMAP, 10)).toEqual([]); + + store.transaction(() => { store.evictEnvelopesBelow(JMAP, '2026-09-01T00:00:00.000Z'); }); + expect(store.countEnvelopes(JMAP)).toBe(0); + expect(store.countBodies(JMAP)).toBe(0); + expect(store.orphanBodies(JMAP, 10)).toEqual([]); + }); + }); + + describe('purge', () => { + it('purgeAll takes the CURSORS with the records', () => { + // A record wipe that leaves a live cursor behind is the one state no amount + // of syncing repairs: /changes cannot re-deliver mail that already existed + // when the cursor was captured. + seed(store); + store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); }); + store.transaction(() => { store.purgeAll(); }); + expect(store.countEnvelopes(JMAP)).toBe(0); + expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull(); + expect(store.getCoverage(JMAP)).toBeNull(); + }); + + it('a wrong key is treated as unreadable and rebuilt, never as a prompt', () => { + store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); }); + store.close(); + const other = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key: randomBytes(32) }); + try { + expect(other.countEnvelopes(JMAP)).toBe(0); + } finally { + other.close(); + } + store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key }); + }); + }); + + describe('policy', () => { + it('round-trips and clamps', () => { + store.transaction(() => { store.setPolicy({ envelopeDays: 99999, bodyDays: 0, maxBodyMB: 1 }); }); + const policy = store.getPolicy(); + expect(policy.envelopeDays).toBe(3650); + expect(policy.bodyDays).toBe(1); + expect(policy.maxBodyMB).toBe(16); + }); + + it('defaults when nothing was ever written', () => { + expect(store.getPolicy()).toEqual(DEFAULT_POLICY); + }); + }); + + describe('read path', () => { + it('lists a mailbox page newest-first with a correct total', () => { + store.transaction(() => { + store.upsertEnvelopes([ + envelope('a', '2026-08-01T00:00:00.000Z'), + envelope('b', '2026-08-02T00:00:00.000Z'), + envelope('c', '2026-08-03T00:00:00.000Z', { mailboxIds: ['archive'] }), + ], 1); + }); + const inbox = store.listEnvelopes(JMAP, 'inbox', 10, 0); + expect(inbox.total).toBe(2); + expect(inbox.rows.map((r) => String(r.id))).toEqual(['b', 'a']); + // A null mailbox is "everything", which is what the unified views want. + expect(store.listEnvelopes(JMAP, null, 10, 0).total).toBe(3); + expect(store.listEnvelopes(JMAP, 'archive', 10, 0).rows.map((r) => String(r.id))).toEqual(['c']); + }); + + it('reports the account ids it holds without needing a network session', () => { + store.transaction(() => { store.upsertEnvelopes([envelope('a', '2026-08-01T00:00:00.000Z')], 1); }); + expect(store.knownJmapAccountIds()).toEqual([JMAP]); + }); + }); +}); + +describe('clampPolicy', () => { + it('never lets the body window exceed the envelope window', () => { + expect(clampPolicy({ envelopeDays: 30, bodyDays: 365, maxBodyMB: 100 }).bodyDays).toBe(30); + }); + + it('falls back to defaults for junk input', () => { + expect(clampPolicy({ envelopeDays: NaN } as never).envelopeDays).toBe(DEFAULT_POLICY.envelopeDays); + expect(clampPolicy(null)).toEqual(DEFAULT_POLICY); + expect(clampPolicy(undefined)).toEqual(DEFAULT_POLICY); + }); +}); + +function seed(store: ReplicaStore): void { + store.transaction(() => { + for (const type of ['Email', 'Mailbox'] as const) { + store.seedCursor( + { jmapAccountId: JMAP, type }, + mintEnumerationCommitment({ + jmapAccountId: JMAP, + snapshot: asSnapshotState('snap'), + targetFrom: '2026-01-01T00:00:00.000Z', + sweepFloor: '2026-01-01T00:00:00.000Z', + kind: 'bootstrap', + }), + 1000, + ); + } + }); +} diff --git a/lib/offline-replica/apply.ts b/lib/offline-replica/apply.ts new file mode 100644 index 00000000..12ee7225 --- /dev/null +++ b/lib/offline-replica/apply.ts @@ -0,0 +1,155 @@ +// Change-application planning. PURE: no network, no storage, no store access. +// +// That purity is the single highest-leverage constraint in the design, because +// `plan(page, presentIds) -> what to fetch` is assertable as plain data. It is +// what turns a failure-mode table into a test suite rather than a promise. + +import type { ChangesState } from './states'; + +export interface ChangesPage { + oldState: ChangesState; + newState: ChangesState; + hasMoreChanges: boolean; + created: string[]; + updated: string[]; + destroyed: string[]; + /** `Mailbox/changes` only. `null`/absent means "assume everything changed". */ + updatedProperties?: string[] | null; +} + +/** + * Collapses the overlap RFC 8620 permits, BEFORE anything iterates - so + * downstream code cannot get the order wrong by following the server's array + * order. + */ +export function normalisePage(page: ChangesPage): { + created: string[]; + updated: string[]; + destroyed: string[]; +} { + const destroyed = [...new Set(page.destroyed)]; + const destroyedSet = new Set(destroyed); + // An id in `destroyed` wins outright: fetching it would be wasted and the + // result would be `notFound`. + const created = [...new Set(page.created)].filter((id) => !destroyedSet.has(id)); + const createdSet = new Set(created); + // An id in both `created` and `updated` is a CREATE - the create path fetches + // the full envelope tier, which already includes the updated values. + const updated = [...new Set(page.updated)].filter( + (id) => !destroyedSet.has(id) && !createdSet.has(id), + ); + return { created, updated, destroyed }; +} + +/** + * An empty page STILL ADVANCES THE CURSOR. Skipping it re-requests the same + * position forever. + */ +export function pageIsEmpty(page: ChangesPage): boolean { + return page.created.length === 0 && page.updated.length === 0 && page.destroyed.length === 0; +} + +export interface EmailFetchPlan { + /** Full envelope tier. */ + createIds: string[]; + /** THREE properties only: id, keywords, mailboxIds. Never bodies. */ + updateIds: string[]; + destroyIds: string[]; +} + +/** + * `keywords` and `mailboxIds` are the ONLY mutable Email properties + * (RFC 8621 s4.1). Body structure, body values, attachments, headers, + * `receivedAt`, `size`, `threadId`, `preview`, `subject`, addresses and + * `hasAttachment` are all immutable for the lifetime of the id. + * + * So an `updated` Email cannot have a changed body, and re-fetching one is pure + * waste. This is also what stops a message cached while unread from staying + * unread forever. + * + * An `updated` id we do NOT hold locally is an UNCONDITIONAL NO-OP, filtered out + * BEFORE the fetch is issued. Cheaper, and it avoids having to fabricate a + * `receivedAt` that a 3-property response cannot supply and the schema's NOT NULL + * would reject. Safe to ignore because absence is always either "retention + * decided against it" or "coverage has not reached it yet" - and coverage + * enumerates CURRENT state, so it will pick the record up with the updated values + * anyway. Nothing needs the update replayed. + */ +export function planEmailFetches( + page: ChangesPage, + presentIds: ReadonlySet, +): EmailFetchPlan { + const { created, updated, destroyed } = normalisePage(page); + return { + createIds: created, + updateIds: updated.filter((id) => presentIds.has(id)), + destroyIds: destroyed, + }; +} + +const COUNT_PROPERTIES = new Set([ + 'totalEmails', 'unreadEmails', 'totalThreads', 'unreadThreads', +]); + +/** + * True when a `Mailbox/changes` update touched only the four counters, so a + * four-integer patch is enough instead of re-fetching every folder object. + * + * `updatedProperties: null` means the server will not say, so everything must be + * re-fetched. An EMPTY array means "nothing but the state token moved", which is + * counts-only vacuously. + */ +export function updatedPropertiesAreCountsOnly( + updatedProperties: readonly string[] | null | undefined, +): boolean { + if (!updatedProperties) return false; + if (updatedProperties.length === 0) return true; + return updatedProperties.every((p) => COUNT_PROPERTIES.has(p)); +} + +export interface MailboxFetchPlan { + /** Needs the whole object. */ + fullIds: string[]; + /** Only the count columns move. */ + countOnlyIds: string[]; + destroyIds: string[]; +} + +export function planMailboxFetches(page: ChangesPage): MailboxFetchPlan { + const { created, updated, destroyed } = normalisePage(page); + const countsOnly = updatedPropertiesAreCountsOnly(page.updatedProperties); + return { + fullIds: countsOnly ? created : [...created, ...updated], + countOnlyIds: countsOnly ? updated : [], + destroyIds: destroyed, + }; +} + +/** + * Keyset-walk progress test. + * + * `after` is INCLUSIVE - this is specified, not implementation-defined. + * RFC 8621 s4.4.1: the `receivedAt` of the Email "must be the same or after this + * date-time to match the condition". So every page after the first re-returns the + * boundary message(s); dedupe by id on commit makes that free. But forward + * progress therefore requires `max(receivedAt)` STRICTLY GREATER than the cursor. + * + * Treating `after` as exclusive and adding a millisecond, as an earlier revision + * of the mobile design did, silently skips every message sharing the boundary + * millisecond on any conforming server. + */ +export function madeForwardProgress( + maxReceivedAt: string | null, + scanCursor: string | null, +): boolean { + if (maxReceivedAt === null) return false; + if (scanCursor === null) return true; + return maxReceivedAt > scanCursor; +} + +/** Advance a scan cursor by exactly one millisecond. The last-resort paging rung. */ +export function advanceOneMs(iso: string): string { + const t = Date.parse(iso); + if (!Number.isFinite(t)) return iso; + return new Date(t + 1).toISOString(); +} diff --git a/lib/offline-replica/engine.ts b/lib/offline-replica/engine.ts new file mode 100644 index 00000000..e00a94cb --- /dev/null +++ b/lib/offline-replica/engine.ts @@ -0,0 +1,190 @@ +// Session resolution, store lifecycle and single-flight. The thin layer every +// `/api/offline/*` replica route goes through. + +import { logger } from '@/lib/logger'; +import { fetchJmapSession, accountIdFor, CAP_MAIL, type JmapSessionInfo } from '@/lib/mail-index/jmap'; +import { withIndexKey } from '@/lib/mail-index/key'; +import { getStoreDir } from '@/lib/mail-index/paths'; +import { + IndexSessionError, resolveIndexSession, type IndexSession, +} from '@/lib/mail-index/reindex'; +import { classify, ReplicaSyncError } from './errors'; +import { ReplicaStore, type RetentionPolicy } from './store'; +import { BUDGET, runCycle, type CycleReport } from './sync'; + +export { IndexSessionError, resolveIndexSession }; +export type { IndexSession }; + +/** + * Opens the replica for one operation and closes it afterwards. + * + * The key is fetched from the main process over the inherited fd for the duration + * of the call only and zeroed after (`withIndexKey`) - there is no cached handle + * and no resident key. A keychain round trip costs microseconds against work that + * makes network calls. + */ +export async function withReplica( + accountId: string, + fn: (store: ReplicaStore) => Promise | T, +): Promise { + const storeDir = getStoreDir(); + if (!storeDir) { + throw new IndexSessionError('The offline replica is not enabled in this deployment.', 404); + } + return withIndexKey(accountId, async (key) => { + const store = ReplicaStore.open({ storeDir, accountId, key }); + try { + return await fn(store); + } finally { + store.close(); + } + }); +} + +/** + * The JMAP account whose mail is replicated. + * + * v1 replicates the PRIMARY mail account only. Every primary key already carries + * `jmap_account_id`, so adding the delegated/shared accounts a single login also + * exposes is inserting rows rather than a migration - JMAP ids are unique only + * WITHIN an account, and a schema that merged them would be cross-account leakage + * that costs nothing to prevent today and is unfixable later. + */ +export function primaryMailAccountId(session: JmapSessionInfo): string | null { + return accountIdFor(session, CAP_MAIL); +} + +/** + * Single-flight per local account. + * + * On `globalThis` rather than in module scope for the same reason + * `lib/mail-index/key.ts` keeps its channel there: Next re-evaluates route + * modules (dev HMR, and separate module instances across route bundles), so a + * module-scoped map is not once-per-process and two overlapping requests would + * each get their own "single" flight. A Symbol key on globalThis is the one place + * in a Node process that survives module re-evaluation. + */ +const FLIGHT_KEY = Symbol.for('vncmail.offlineReplica.inFlight'); + +function flights(): Map> { + const holder = globalThis as unknown as Record> | undefined>; + const existing = holder[FLIGHT_KEY]; + if (existing) return existing; + const created = new Map>(); + holder[FLIGHT_KEY] = created; + return created; +} + +export interface SyncOptions { + /** Overrides the persisted policy for this cycle, and persists the override. */ + policy?: RetentionPolicy; + /** Forces a rebuild: sets the sticky resync flag before the cycle runs. */ + forceResync?: boolean; +} + +/** + * Runs one cycle for the calling session's account, coalescing concurrent callers + * onto the same promise. + * + * Coalescing rather than aborting is deliberate: an implementation that set an + * abort flag and returned produced a cancelled sync and no new one - a "Sync now" + * tap during a sync did nothing at all. The in-flight promise is assigned to the + * map BEFORE the cycle body runs, because several early-return paths resolve + * synchronously and a later assignment leaves a re-entrancy hole; the cleanup is + * identity-checked so a slow loser cannot delete a newer flight. + */ +export async function syncAccount( + indexSession: IndexSession, + options: SyncOptions = {}, +): Promise { + const map = flights(); + const existing = map.get(indexSession.accountId); + if (existing) return existing; + + const run = (async (): Promise => { + // The session fetch is the FIRST network call of a cycle, so when the backend + // is unreachable this is where it fails - and it must be classified by the same + // taxonomy as everything else. Found by execution: without this, an offline + // sync surfaced a bare `JmapIndexError` 502 with no error class, so a caller + // could not tell "the network is down, retry later" from "this deployment is + // broken". "Offline is not an error" has to hold at the very first call too. + let session: JmapSessionInfo; + try { + session = await fetchJmapSession(indexSession.serverUrl, indexSession.authHeader); + } catch (error) { + const status = (error as { status?: number } | null)?.status; + const message = error instanceof Error ? error.message : String(error); + // `JmapIndexError.status` is OUR OWN value, not a server response status: + // it is 401 for auth, 429 for rate limiting, 504 for a timeout, and 502 for + // everything else - INCLUDING a `fetch` rejection with no server involved at + // all. So a bare 502 must be classified from the message, or "the machine is + // offline" is misread as "the server returned a 5xx". Passing the synthetic + // status straight into `classify` produced exactly that, found by the + // network-cut integration run. + if (status === 401 || status === 403) throw error; + const cls = + status === 429 ? 'RateLimit' as const + : status === 504 ? 'Transport' as const + : classify({ message }); + throw new ReplicaSyncError(cls, message); + } + const jmapAccountId = primaryMailAccountId(session); + if (!jmapAccountId) { + throw new IndexSessionError('This account has no JMAP mail capability.', 409); + } + + return withReplica(indexSession.accountId, async (store) => { + const now = Date.now(); + if (options.policy) store.transaction(() => { store.setPolicy(options.policy as RetentionPolicy); }); + if (options.forceResync) { + store.transaction(() => { store.patchFlags(now, { resyncRequired: true }); }); + } + const policy = store.getPolicy(); + const report = await runCycle({ + store, + session, + authHeader: indexSession.authHeader, + jmapAccountId, + policy, + now, + deadline: now + BUDGET.wallClockMs, + }); + logger.info('offline-replica: cycle complete', { + slot: indexSession.slot, + ok: report.ok, + phase: report.coveragePhase, + envelopes: report.envelopesWritten, + bodies: report.bodiesWritten, + deleted: report.envelopesDeleted, + unfinished: report.unfinishedWork, + warnings: report.warnings.length, + durationMs: report.durationMs, + }); + return report; + }); + })(); + + map.set(indexSession.accountId, run); + try { + return await run; + } finally { + if (map.get(indexSession.accountId) === run) map.delete(indexSession.accountId); + } +} + +/** + * Resolves the JMAP account id for a READ without any network call. + * + * The read path must work with the backend unreachable, so it cannot fetch a JMAP + * session to learn the primary account id - that fetch is exactly what fails when + * offline. The store knows which account ids it holds rows for; with one + * replicated account that is unambiguous, and the caller may also pass an explicit + * id. + */ +export function resolveReadAccountId(store: ReplicaStore, requested?: string | null): string | null { + const known = store.knownJmapAccountIds(); + if (requested && known.includes(requested)) return requested; + if (known.length === 1) return known[0]; + if (requested) return null; + return known[0] ?? null; +} diff --git a/lib/offline-replica/errors.ts b/lib/offline-replica/errors.ts new file mode 100644 index 00000000..bd976409 --- /dev/null +++ b/lib/offline-replica/errors.ts @@ -0,0 +1,151 @@ +// The error taxonomy. The whole point of this file is one column of one table: +// **exactly one error class moves the cursor**, and its action is a full +// verified rebuild. Everywhere else, failure means the cursor stands still. +// +// That is what makes "a failure never causes silent data loss" structural rather +// than aspirational - and it is precisely what the mobile client's shipped +// defect D4 got wrong, by collapsing every error to `null` and then adopting a +// snapshot state as the next cursor. A transient 503 on `Email/changes` was +// enough to fast-forward the cursor over every change the client had not seen. + +export type ErrorClass = + | 'Transport' + | 'RateLimit' + | 'ServerTransient' + | 'RequestLimit' + | 'Auth' + | 'Fatal' + | 'StateInvalid'; + +/** True for the one class that moves a cursor - and it moves it to "invalidated". */ +export function movesCursor(cls: ErrorClass): boolean { + return cls === 'StateInvalid'; +} + +/** + * Only a size/availability problem is worth escalating to a rebuild. + * + * Escalating on RateLimit would mean the response to a rate-limited server is to + * issue far MORE requests - a full window re-enumeration. Escalating on Auth + * would let a 401 trigger a rebuild; on Transport, a flaky tunnel would do the + * same. Fatal is our own bug and a rebuild will not fix it. + */ +export function escalationApplies(cls: ErrorClass): boolean { + return cls === 'ServerTransient' || cls === 'RequestLimit'; +} + +/** JMAP method-level error types that invalidate a `/changes` cursor. */const STATE_INVALID_TYPES = new Set(['cannotCalculateChanges']); + +const FATAL_TYPES = new Set([ + 'invalidArguments', 'unknownMethod', 'accountNotFound', 'forbidden', + 'unsupportedFilter', 'unsupportedSort', 'invalidResultReference', + 'accountNotSupportedByMethod', 'accountReadOnly', +]); + +const REQUEST_LIMIT_TYPES = new Set([ + 'maxSizeRequest', 'maxCallsInRequest', 'requestTooLarge', 'maxObjectsInGet', + 'tooLarge', +]); + +const SERVER_TRANSIENT_TYPES = new Set([ + 'serverUnavailable', 'serverFail', 'serverPartialFail', 'stateMismatch', +]); + +export class ReplicaSyncError extends Error { + readonly cls: ErrorClass; + readonly retryAfterMs?: number; + constructor(cls: ErrorClass, message: string, retryAfterMs?: number) { + super(message); + this.name = 'ReplicaSyncError'; + this.cls = cls; + this.retryAfterMs = retryAfterMs; + } +} + +/** + * Classification is STRUCTURE BEFORE STRINGS: HTTP status, then JMAP error type, + * and only then message prose. A method error's `description` can legitimately + * contain the words "timeout" or "socket", and `fetch failed: ECONNRESET` must + * not be read as a JMAP method error. + */ +export function classify(input: { + httpStatus?: number; + jmapErrorType?: string; + message?: string; +}): ErrorClass { + const { httpStatus, jmapErrorType, message } = input; + + if (typeof httpStatus === 'number') { + if (httpStatus === 401 || httpStatus === 403) return 'Auth'; + if (httpStatus === 429) return 'RateLimit'; + if (httpStatus === 413) return 'RequestLimit'; + if (httpStatus >= 500) return 'ServerTransient'; + } + + if (jmapErrorType) { + if (STATE_INVALID_TYPES.has(jmapErrorType)) return 'StateInvalid'; + if (REQUEST_LIMIT_TYPES.has(jmapErrorType)) return 'RequestLimit'; + if (FATAL_TYPES.has(jmapErrorType)) return 'Fatal'; + if (SERVER_TRANSIENT_TYPES.has(jmapErrorType)) return 'ServerTransient'; + if (jmapErrorType === 'limit') return 'RateLimit'; + // An UNRECOGNISED method-level error is ServerTransient, never Fatal and + // never StateInvalid. Guessing transient costs a retry; guessing + // state-invalid costs a full resync; guessing fatal stalls the account. The + // cheapest wrong answer wins the default. + return 'ServerTransient'; + } + + if (message) { + const lower = message.toLowerCase(); + if ( + lower.includes('fetch failed') || lower.includes('econnrefused') || + lower.includes('econnreset') || lower.includes('enotfound') || + lower.includes('etimedout') || lower.includes('socket') || + lower.includes('network') || lower.includes('timed out') || + lower.includes('eai_again') || lower.includes('ehostunreach') || + lower.includes('enetunreach') || lower.includes('certificate') + ) { + // "Offline is not an error." Transport failures leave every cursor exactly + // where it was and are retried later. + return 'Transport'; + } + } + + return 'ServerTransient'; +} + +/** Full-jitter exponential backoff. */ +export function backoffDelayMs(attempt: number, opts: { baseMs?: number; capMs?: number } = {}): number { + const base = opts.baseMs ?? 1_000; + const cap = opts.capMs ?? 60_000; + const ceiling = Math.min(cap, base * 2 ** Math.max(0, attempt)); + // Jitter is not decoration: several triggers fire at once (launch catch-up, + // network recovery, a push burst) against one Stalwart instance, which is + // exactly the shape that produces a synchronised stampede. + return Math.floor(Math.random() * ceiling); +} + +/** + * The `maxChanges` ladder, monotonically SHRINKING, every rung expressed + * relative to rung 0. + * + * Two bugs live here historically. First, an unbounded middle rung produced a + * retry strictly LARGER than the attempt that just failed - actively worsening a + * "response too large" error. Then clamping only rung 0 reintroduced it in a + * narrower form: a server advertising `maxObjectsInGet: 100` gave rung 0 = 100 + * and rung 1 = 250. Deriving every rung from rung 0 is what makes + * monotonic-non-increase true for every server value. + */ +export function rungValue(rung: 0 | 1 | 2 | 3, maxObjectsInGet: number | undefined): number { + const rung0 = Math.max(1, Math.min(maxObjectsInGet ?? 500, 500)); + switch (rung) { + case 0: return rung0; + case 1: return Math.max(1, Math.min(rung0, 250)); + case 2: return Math.max(1, Math.min(rung0, 50)); + case 3: return Math.max(1, Math.min(rung0, 25)); + } +} + +export function nextRung(rung: 0 | 1 | 2 | 3): 0 | 1 | 2 | 3 { + return rung >= 3 ? 3 : ((rung + 1) as 0 | 1 | 2 | 3); +} diff --git a/lib/offline-replica/jmap.ts b/lib/offline-replica/jmap.ts new file mode 100644 index 00000000..874425f7 --- /dev/null +++ b/lib/offline-replica/jmap.ts @@ -0,0 +1,302 @@ +// The replica's JMAP calls. Reuses `lib/mail-index/jmap.ts`'s session fetch, +// origin pinning and request plumbing rather than duplicating them (that file +// already handles Stalwart's 307 on /.well-known/jmap and refuses to send +// credentials off-origin), and adds the delta-sync methods the index never +// needed: `Mailbox/changes`, `Email/changes`, the ascending coverage query, and +// the two-tier `Email/get`. +// +// THIS FILE IS THE ONLY PLACE ALLOWED TO MINT A BRANDED STATE TOKEN. That is what +// makes cursor provenance checkable by grep: `asChangesState` appears only in the +// `/changes` parser, `asSnapshotState` only in the `Foo/get {ids: []}` parser. + +import type { Email, Mailbox } from '@/lib/jmap/types'; +import { CAP_CORE, CAP_MAIL, jmapRequest, type JmapSessionInfo } from '@/lib/mail-index/jmap'; +import type { ChangesPage } from './apply'; +import { ReplicaSyncError, classify } from './errors'; +import { asChangesState, asSnapshotState, type SnapshotState } from './states'; + +/** The envelope tier. Mirrors `lib/jmap/client.ts`'s EMAIL_LIST_PROPERTIES exactly. */ +export const ENVELOPE_PROPERTIES = [ + 'id', 'threadId', 'mailboxIds', 'keywords', 'size', 'receivedAt', + 'from', 'to', 'cc', 'subject', 'preview', 'hasAttachment', 'blobId', +] as const; + +/** + * The body tier - everything `lib/jmap/client.ts`'s `getEmail()` asks for beyond + * the envelope tier, so a replica-served message is field-for-field what the + * online read path produces. `components/email/email-viewer.tsx` reads + * `bodyValues` keyed by the SAME partIds as `htmlBody`/`textBody`, so all three + * must travel together or the viewer sits on its loading skeleton forever. + */ +export const BODY_PROPERTIES = [ + 'id', 'sentAt', 'bcc', 'replyTo', 'textBody', 'htmlBody', 'bodyValues', + 'attachments', 'messageId', 'inReplyTo', 'references', 'headers', 'bodyStructure', +] as const; + +/** The three MUTABLE properties. */ +export const MUTABLE_PROPERTIES = ['id', 'keywords', 'mailboxIds'] as const; + +export const MAX_BODY_VALUE_BYTES = 512_000; + +interface MethodError { + type?: string; + description?: string; +} + +function asMethodError(args: Record): MethodError { + return { + type: typeof args.type === 'string' ? args.type : undefined, + description: typeof args.description === 'string' ? args.description : undefined, + }; +} + +/** + * Runs one JMAP request and classifies any failure. Wraps the shared transport so + * a transport-level failure becomes `Transport` (cursor untouched) rather than an + * opaque throw the caller has to guess about. + */ +async function call( + session: JmapSessionInfo, + authHeader: string, + methodCalls: ReadonlyArray<[string, Record, string]>, +): Promise, string]>> { + try { + return await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], methodCalls); + } catch (error) { + const status = (error as { status?: number } | null)?.status; + const message = error instanceof Error ? error.message : String(error); + throw new ReplicaSyncError(classify({ httpStatus: status, message }), message); + } +} + +function findResponse( + responses: Array<[string, Record, string]>, + callId: string, +): { name: string; args: Record } | null { + for (const [name, args, id] of responses) { + if (id === callId) return { name, args }; + } + return null; +} + +/** Turns a method-level `error` response into a classified throw. */ +function raiseMethodError(args: Record, context: string): never { + const { type, description } = asMethodError(args); + throw new ReplicaSyncError( + classify({ jmapErrorType: type, message: description }), + `${context} failed: ${type ?? 'unknown'}${description ? ` (${description})` : ''}`, + ); +} + +function strArray(v: unknown): string[] { + return Array.isArray(v) ? v.filter((x): x is string => typeof x === 'string') : []; +} + +// ── snapshot states, for bootstrap / reconcile ─────────────────────────────── + +/** + * Captures both cursors in ONE request, before touching any data. + * + * `Foo/get {ids: []}` returns the account's current state token with no records, + * which RFC 8620 s5.1 defines as a valid `sinceState` for `Foo/changes`. This is + * step 1 of the mandatory bootstrap order and the single thing most likely to be + * "optimised" into a permanent data hole: the cursor must be captured BEFORE the + * enumeration, so it is deliberately OLDER than the data and the first delta cycle + * re-delivers a few changes we already have. The cheaper opposite order - enumerate, + * then capture - silently loses every change that arrived during the scan, which on + * a large mailbox is minutes. + */ +export async function captureSnapshotStates( + session: JmapSessionInfo, + authHeader: string, + accountId: string, +): Promise<{ mailbox: SnapshotState; email: SnapshotState }> { + const responses = await call(session, authHeader, [ + ['Mailbox/get', { accountId, ids: [] }, 'm'], + ['Email/get', { accountId, ids: [] }, 'e'], + ]); + const mailbox = findResponse(responses, 'm'); + const email = findResponse(responses, 'e'); + if (!mailbox || mailbox.name === 'error') { + raiseMethodError(mailbox?.args ?? {}, 'Mailbox/get (state capture)'); + } + if (!email || email.name === 'error') { + raiseMethodError(email?.args ?? {}, 'Email/get (state capture)'); + } + return { + mailbox: asSnapshotState(mailbox.args.state), + email: asSnapshotState(email.args.state), + }; +} + +// ── /changes ───────────────────────────────────────────────────────────────── + +function parseChangesPage(args: Record): ChangesPage { + return { + oldState: asChangesState(args.oldState), + newState: asChangesState(args.newState), + hasMoreChanges: args.hasMoreChanges === true, + created: strArray(args.created), + updated: strArray(args.updated), + destroyed: strArray(args.destroyed), + updatedProperties: + args.updatedProperties === null + ? null + : Array.isArray(args.updatedProperties) + ? strArray(args.updatedProperties) + : undefined, + }; +} + +export async function getMailboxChanges( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + sinceState: string, + maxChanges: number, +): Promise { + const responses = await call(session, authHeader, [ + ['Mailbox/changes', { accountId, sinceState, maxChanges }, 'c'], + ]); + const res = findResponse(responses, 'c'); + if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Mailbox/changes'); + return parseChangesPage(res.args); +} + +export async function getEmailChanges( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + sinceState: string, + maxChanges: number, +): Promise { + const responses = await call(session, authHeader, [ + ['Email/changes', { accountId, sinceState, maxChanges }, 'c'], + ]); + const res = findResponse(responses, 'c'); + if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Email/changes'); + // `Email/changes` has no `updatedProperties` - RFC 8621 s4.3 is a plain + // /changes - which is why the 3-property `Email/get` is unavoidable there. + return parseChangesPage(res.args); +} + +// ── gets ───────────────────────────────────────────────────────────────────── + +export async function getMailboxes( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + ids: readonly string[] | null, + properties?: readonly string[], +): Promise { + const args: Record = { accountId, ids: ids === null ? null : [...ids] }; + if (properties) args.properties = [...properties, 'id']; + const responses = await call(session, authHeader, [['Mailbox/get', args, 'g']]); + const res = findResponse(responses, 'g'); + if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Mailbox/get'); + return Array.isArray(res.args.list) ? (res.args.list as Mailbox[]) : []; +} + +export interface EmailGetResult { + list: Email[]; + /** Normal, not an error: the record was destroyed between /changes and /get. */ + notFound: string[]; +} + +export async function getEmails( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + ids: readonly string[], + tier: 'envelope' | 'mutable' | 'body', +): Promise { + if (ids.length === 0) return { list: [], notFound: [] }; + const args: Record = { accountId, ids: [...ids] }; + if (tier === 'envelope') { + args.properties = [...ENVELOPE_PROPERTIES]; + } else if (tier === 'mutable') { + args.properties = [...MUTABLE_PROPERTIES]; + } else { + args.properties = [...BODY_PROPERTIES]; + // Without these the bodyValues map comes back EMPTY and every stored body + // would be an empty object that renders as a blank message offline. + args.fetchTextBodyValues = true; + args.fetchHTMLBodyValues = true; + args.fetchAllBodyValues = true; + args.maxBodyValueBytes = MAX_BODY_VALUE_BYTES; + } + const responses = await call(session, authHeader, [['Email/get', args, 'g']]); + const res = findResponse(responses, 'g'); + if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Email/get'); + return { + list: Array.isArray(res.args.list) ? (res.args.list as Email[]) : [], + notFound: strArray(res.args.notFound), + }; +} + +// ── coverage enumeration ───────────────────────────────────────────────────── + +export interface CoveragePage { + ids: string[]; + /** Echoed back so the caller can detect the tie-cluster case. */ + requestedAfter: string; +} + +/** + * The ascending keyset walk. + * + * ASCENDING is not a style choice. New mail arrives at the TAIL, so insertions + * never shift rows the scan has already passed. With a DESCENDING sort and + * position-based paging, one delivery between page 1 and page 2 pushes a message + * from page 1's boundary into page 2's start and one message out of the scan's + * reach entirely - and that message is pre-existing relative to our cursor, so + * `Email/changes` will never report it. A permanent hole with no signal it exists. + * + * `calculateTotal: false` because the total is unstable and unused. + */ +export async function queryAscending( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + after: string, + limit: number, + anchor?: { anchor: string; anchorOffset: number }, +): Promise { + const args: Record = { + accountId, + filter: { after }, + sort: [{ property: 'receivedAt', isAscending: true }], + limit, + calculateTotal: false, + }; + if (anchor) { + args.anchor = anchor.anchor; + args.anchorOffset = anchor.anchorOffset; + } + const responses = await call(session, authHeader, [['Email/query', args, 'q']]); + const res = findResponse(responses, 'q'); + if (!res || res.name === 'error') { + const { type } = asMethodError(res?.args ?? {}); + if (type === 'anchorNotFound') { + // Not a failure - the caller falls back to the last-resort rung. + throw new AnchorNotFoundError(); + } + raiseMethodError(res?.args ?? {}, 'Email/query'); + } + return { ids: strArray(res.args.ids), requestedAfter: after }; +} + +export class AnchorNotFoundError extends Error { + constructor() { + super('Email/query rejected the anchor'); + this.name = 'AnchorNotFoundError'; + } +} + +/** `maxObjectsInGet`, so the maxChanges ladder can be clamped to what the server allows. */ +export function maxObjectsInGet(session: JmapSessionInfo): number | undefined { + const core = session.capabilities[CAP_CORE]; + if (!core || typeof core !== 'object') return undefined; + const value = (core as Record).maxObjectsInGet; + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} diff --git a/lib/offline-replica/read.ts b/lib/offline-replica/read.ts new file mode 100644 index 00000000..c0248d56 --- /dev/null +++ b/lib/offline-replica/read.ts @@ -0,0 +1,219 @@ +// The offline READ path: stored rows back into the exact `Email` / `Mailbox` +// shapes `lib/jmap/client.ts` returns, so the renderer cannot tell the difference. +// +// COHERENCE (the review's H3, the one finding that genuinely returns once a +// replica exists). The webmail already does LOCAL DELTA ARITHMETIC on mailbox +// unread counts and totals for mark-read/move/delete, with a comment referencing +// a production bug from getting that cutoff wrong. A read-only cache sitting +// underneath that arithmetic needs an explicit coherence story, and the story is: +// +// THE REPLICA IS A FALLBACK, NEVER A CACHE IN FRONT OF THE SERVER. +// +// It is consulted only after a read has actually failed at the transport level +// (see `lib/offline-fallback-client.ts`), so an online session never sees a +// replica count and the arithmetic never operates on replica numbers. While +// offline, counts are whatever the last successful sync recorded and any local +// mark-read drift is bounded, invisible in the same session, and repaired by the +// next `Mailbox/changes` - which is the authoritative source for all four +// counters. The alternative - serving the replica first and reconciling - is what +// would need the coherence rules the review asked for, and is not what this does. +// +// Every shape here is intentionally what the ONLINE path produces, including +// `parseEmailHeaders`' derived security fields, because +// `components/email/email-viewer.tsx` reads them directly. In particular +// `bodyValues` must be keyed by the same partIds as `htmlBody`/`textBody`, or the +// viewer's `isBodyLoading` gate sits on its skeleton forever. + +import { parseAuthenticationResults, parseSpamLLM, parseSpamScore } from '@/lib/email-headers'; +import type { Email, EmailAddress, Mailbox } from '@/lib/jmap/types'; +import type { ReplicaStore } from './store'; +import type { MailboxRow } from './types'; + +const DEFAULT_RIGHTS: Mailbox['myRights'] = { + mayReadItems: true, mayAddItems: false, mayRemoveItems: false, maySetSeen: false, + maySetKeywords: false, mayCreateChild: false, mayRename: false, mayDelete: false, + maySubmit: false, +}; + +function parseJson(raw: unknown, fallback: T): T { + if (typeof raw !== 'string' || raw.length === 0) return fallback; + try { + return JSON.parse(raw) as T; + } catch { + return fallback; + } +} + +export function rowToMailbox(row: MailboxRow): Mailbox { + return { + id: row.id, + name: row.name, + parentId: row.parentId ?? undefined, + role: row.role ?? undefined, + sortOrder: row.sortOrder ?? 0, + totalEmails: row.totalEmails ?? 0, + unreadEmails: row.unreadEmails ?? 0, + totalThreads: row.totalThreads ?? 0, + unreadThreads: row.unreadThreads ?? 0, + // Offline, the rights that matter are the read ones. Every mutating right + // defaults to false so no UI offers an action that cannot possibly succeed + // with no network; the real rights return with the next sync. + myRights: parseJson(row.myRightsJson, DEFAULT_RIGHTS), + isSubscribed: row.isSubscribed, + }; +} + +/** The envelope tier, as `getEmails()` would return it. */ +export function rowToEnvelope(row: Record, mailboxIds: readonly string[]): Email { + const keywords = parseJson>(row.keywords_json, {}); + const mailboxMap: Record = {}; + for (const id of mailboxIds) mailboxMap[id] = true; + return { + id: String(row.id), + threadId: typeof row.thread_id === 'string' ? row.thread_id : String(row.id), + mailboxIds: mailboxMap, + keywords, + size: typeof row.size === 'number' ? row.size : 0, + receivedAt: String(row.received_at), + from: parseJson(row.from_json, undefined), + to: parseJson(row.to_json, undefined), + cc: parseJson(row.cc_json, undefined), + subject: typeof row.subject === 'string' ? row.subject : undefined, + preview: typeof row.preview === 'string' ? row.preview : undefined, + hasAttachment: row.has_attachment !== 0, + blobId: typeof row.blob_id === 'string' ? row.blob_id : undefined, + }; +} + +interface StoredBody { + sentAt?: string; + bcc?: EmailAddress[]; + replyTo?: EmailAddress[]; + textBody?: Email['textBody']; + htmlBody?: Email['htmlBody']; + bodyValues?: Email['bodyValues']; + attachments?: Email['attachments']; + messageId?: string; + inReplyTo?: string[]; + references?: string[]; + headers?: unknown; + bodyStructure?: Email['bodyStructure']; +} + +/** + * Normalises JMAP's `headers` array into the record shape the renderer expects + * and derives the security fields, reproducing what `JMAPClient`'s private + * `parseEmailHeaders` does on the online path. + * + * Reproduced here rather than imported because `lib/jmap/client.ts` is a + * 7400-line renderer object that holds credentials in instance fields, opens push + * connections and wires itself into Zustand stores - importing it into a server + * route would drag all of that into the server bundle. The parsing HELPERS in + * `lib/email-headers` are shared, so the only duplicated logic is the array->record + * flattening. + */ +function applyHeaders(email: Email, rawHeaders: unknown): void { + let record: Record; + if (Array.isArray(rawHeaders)) { + record = {}; + for (const header of rawHeaders as Array<{ name?: string; value?: string }>) { + if (!header?.name || !header?.value) continue; + const existing = record[header.name]; + if (existing) { + record[header.name] = Array.isArray(existing) + ? [...existing, header.value] + : [existing, header.value]; + } else { + record[header.name] = header.value; + } + } + } else if (rawHeaders && typeof rawHeaders === 'object') { + record = rawHeaders as Record; + } else { + return; + } + email.headers = record; + + const authResults = record['Authentication-Results']; + if (authResults) { + const value = Array.isArray(authResults) ? authResults.join('; ') : authResults; + email.authenticationResults = parseAuthenticationResults(value); + } + + for (const name of ['X-Spam-Score', 'X-Spam-Status', 'X-Spam-Result', 'X-Rspamd-Score']) { + const header = record[name]; + if (!header) continue; + const value = Array.isArray(header) ? header[0] : header; + const parsed = parseSpamScore(String(value).trim()); + if (parsed) { + email.spamScore = parsed.score; + email.spamStatus = parsed.status; + break; + } + } + + const llm = record['X-Spam-LLM']; + if (llm) { + const parsed = parseSpamLLM(String(Array.isArray(llm) ? llm[0] : llm)); + if (parsed) email.spamLLM = parsed; + } +} + +export interface OfflineMessage { + email: Email; + /** False when only the envelope is held, so the caller can say so rather than render blank. */ + hasBody: boolean; +} + +/** One full message, envelope + body, exactly as `getEmail()` would return it. */ +export function readMessage( + store: ReplicaStore, + jmapAccountId: string, + id: string, +): OfflineMessage | null { + const row = store.getEnvelopeRaw(jmapAccountId, id); + if (!row) return null; + const email = rowToEnvelope(row, store.mailboxIdsFor(jmapAccountId, id)); + + const bodyJson = store.getBody(jmapAccountId, id); + if (!bodyJson) return { email, hasBody: false }; + + const body = parseJson(bodyJson, {}); + email.sentAt = body.sentAt; + email.bcc = body.bcc; + email.replyTo = body.replyTo; + email.textBody = body.textBody; + email.htmlBody = body.htmlBody; + email.bodyValues = body.bodyValues; + email.attachments = body.attachments; + email.messageId = body.messageId; + email.inReplyTo = body.inReplyTo; + email.references = body.references; + email.bodyStructure = body.bodyStructure; + applyHeaders(email, body.headers); + + // A body row whose `bodyValues` came back empty would render as a blank + // message and, worse, leave the viewer's loading gate stuck. Report it as + // "envelope only" instead, which the UI can explain. + const hasBody = + !!email.bodyValues && Object.keys(email.bodyValues).length > 0; + return { email, hasBody }; +} + +export function readMailboxes(store: ReplicaStore, jmapAccountId: string): Mailbox[] { + return store.listMailboxes(jmapAccountId).map(rowToMailbox); +} + +export function readEnvelopePage( + store: ReplicaStore, + jmapAccountId: string, + mailboxId: string | null, + limit: number, + offset: number, +): { emails: Email[]; total: number; hasMore: boolean } { + const { rows, total } = store.listEnvelopes(jmapAccountId, mailboxId, limit, offset); + const emails = rows.map((row) => + rowToEnvelope(row, store.mailboxIdsFor(jmapAccountId, String(row.id))), + ); + return { emails, total, hasMore: offset + emails.length < total }; +} diff --git a/lib/offline-replica/retention.ts b/lib/offline-replica/retention.ts new file mode 100644 index 00000000..77636cc6 --- /dev/null +++ b/lib/offline-replica/retention.ts @@ -0,0 +1,149 @@ +// Retention floors, and the clock-jump guard. +// +// THE BUG THIS FILE IS SHAPED BY. No cursor and no ordering in this engine +// depends on the device clock - but the retention BOUNDARY does, so a large clock +// skew moves the window. The mobile client added a guard: if the computed floor +// moves more than ~25 h from the last one, keep the previous floor and warn. +// +// The guard then wiped the entire offline store. Mechanism, exactly: +// +// 1. clock jumps forward a year +// 2. cycle N computes a floor a year ahead, detects the >25 h move, holds the +// old floor for this cycle - and PERSISTS THE COMPUTED (jumped) FLOOR as +// `lastWindowFloor` +// 3. the follow-on cycle a few seconds later computes a floor within seconds of +// the persisted one, so `delta <= threshold`, so the guard passes +// 4. that floor is classified as a retention NARROW, and every envelope below +// it is evicted - i.e. all of them, since the floor is a year in the future +// 5. `coveredFrom` then claims the range complete, and `/changes` cannot +// re-deliver pre-existing mail. Unrecoverable. +// +// A clock glitch wiped the store about five seconds after being detected, +// THROUGH the very mechanism meant to prevent that. Its reproduction returned 0 +// envelopes from a 4-envelope store. +// +// The generalisable lesson, worth more than the code: a guard that DETECTS an +// anomaly but PERSISTS the anomalous value converts a transient glitch into a +// legitimised new baseline. Any "suppress and remember" guard must remember the +// value it USED, not the value it rejected - and must expose a separate +// "don't delete anything on this basis" bit, because suppressing the floor is not +// the same as suppressing the deletions the floor authorises. + +import type { RetentionPolicy } from './store'; + +/** A day plus an hour: an ordinary DST shift or NTP correction must not trip it. */ +export const CLOCK_JUMP_GUARD_MS = 25 * 60 * 60 * 1000; + +export interface RetentionFloors { + envelopeFrom: string; + bodyFrom: string; + maxBodyBytes: number; +} + +function isoDaysAgo(now: number, days: number): string { + return new Date(now - days * 24 * 60 * 60 * 1000).toISOString(); +} + +export function computeFloors(policy: RetentionPolicy, now: number): RetentionFloors { + // The body window can never be wider than the envelope window: a body with no + // envelope is an orphan by construction, and the whole point of two tiers is + // envelopes being a superset of bodies. + const bodyDays = Math.min(policy.bodyDays, policy.envelopeDays); + return { + envelopeFrom: isoDaysAgo(now, policy.envelopeDays), + bodyFrom: isoDaysAgo(now, bodyDays), + maxBodyBytes: Math.max(0, Math.floor(policy.maxBodyMB * 1024 * 1024)), + }; +} + +export interface GuardedFloor { + /** The floor to actually use for eviction and for any sweep. */ + envelopeFrom: string; + /** True when the guard suppressed a suspicious jump. */ + suppressed: boolean; + /** + * False while the floor in use came from a SUSPECT clock reading. Retention + * eviction and the reconcile sweep must BOTH refuse to act on it - suppressing + * the floor is not the same as suppressing the deletions it authorises. + */ + evictionAllowed: boolean; + /** What to persist as `lastWindowFloor`. */ + nextLastWindowFloor: string; + warning?: string; +} + +export function guardFloorAgainstClockJump( + computed: string, + lastWindowFloor: string | undefined, + opts: { policyChanged?: boolean } = {}, +): GuardedFloor { + const adopt = (floor: string): GuardedFloor => ({ + envelopeFrom: floor, + suppressed: false, + evictionAllowed: true, + nextLastWindowFloor: floor, + }); + + if (!lastWindowFloor) return adopt(computed); + + // An explicit `envelopeDays` change is INTENT, not a glitch, and must take + // effect - including its eviction. This is also why "adopt on the second + // consistent observation" had to go: with intent handled here, that rule + // existed ONLY for the clock-anomaly case, i.e. only for the case where + // adopting is the harmful thing to do. + if (opts.policyChanged) return adopt(computed); + + const delta = Math.abs(Date.parse(computed) - Date.parse(lastWindowFloor)); + if (!Number.isFinite(delta) || delta <= CLOCK_JUMP_GUARD_MS) return adopt(computed); + + // SUSPECT. Ignore the reading entirely, keep the previous floor, and persist + // THE PREVIOUS FLOOR - so every subsequent cycle re-detects the same jump and + // stays suppressed. Persisting the computed value here is the wipe described + // at the top of this file. + // + // The trade-off, stated rather than hidden: on a device whose clock is + // permanently wrong by more than a day, retention stops tracking the clock and + // the store keeps MORE mail than the setting says. That is bounded (envelopes + // are ~1 KB, bodies are capped in bytes) and self-clears the moment the user + // changes a retention setting or the clock returns. Keeping too much mail is + // the correct direction to fail for a feature whose entire purpose is having + // mail available offline. + return { + envelopeFrom: lastWindowFloor, + suppressed: true, + evictionAllowed: false, + nextLastWindowFloor: lastWindowFloor, + warning: + `retention floor moved ${Math.round(delta / 3_600_000)}h in one step ` + + `(${lastWindowFloor} -> ${computed}); treating it as a clock anomaly and ` + + `refusing to evict or sweep on this basis`, + }; +} + +export type FloorMovement = 'unchanged' | 'widened' | 'narrowed'; + +export function floorMovement(previous: string | undefined, next: string): FloorMovement { + if (!previous) return 'unchanged'; + if (next === previous) return 'unchanged'; + // A LATER floor keeps less mail. + return next > previous ? 'narrowed' : 'widened'; +} + +export interface WindowAdjustment { + /** Set when envelopes below this floor should be evicted. */ + evictBelow?: string; + /** Set when coverage must re-scan from a wider floor. */ + rescanFrom?: string; +} + +/** + * What a floor movement asks for. + * + * A WIDEN is not a resync: `targetFrom` moves back, coverage re-enters + * `scanning`, and the cursors are untouched. A NARROW evicts. + */ +export function adjustForWindow(movement: FloorMovement, floor: string): WindowAdjustment { + if (movement === 'narrowed') return { evictBelow: floor }; + if (movement === 'widened') return { rescanFrom: floor }; + return {}; +} diff --git a/lib/offline-replica/route-gate.ts b/lib/offline-replica/route-gate.ts new file mode 100644 index 00000000..dcb734b2 --- /dev/null +++ b/lib/offline-replica/route-gate.ts @@ -0,0 +1,67 @@ +// The gate every replica route shares, and its error mapping. +// +// 404, not 403, when the feature is absent: the standalone server artifact is the +// SAME one the production Dockerfile ships to multi-tenant deployments, where a +// server-side replica of every user's mail would be badly wrong. Nothing should +// learn the routes exist in a deployment that does not have the feature. + +import { NextResponse } from 'next/server'; +import { logger } from '@/lib/logger'; +import { isSqlcipherAvailable } from '@/lib/mail-index/binding'; +import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key'; +import { getStoreDir } from '@/lib/mail-index/paths'; +import { JmapIndexError } from '@/lib/mail-index/jmap'; +import { IndexSessionError } from '@/lib/mail-index/reindex'; +import { ReplicaUnavailableError } from './store'; +import { ReplicaSyncError } from './errors'; + +/** Returns a response to send immediately, or `null` when the gate is open. */ +export function gateReplicaRoute(): NextResponse | null { + if (!getStoreDir()) return new NextResponse(null, { status: 404 }); + if (!hasKeyChannel()) { + return NextResponse.json( + { error: 'The offline replica has no key channel in this process.', code: 'no-key-channel' }, + { status: 503 }, + ); + } + if (!isSqlcipherAvailable()) { + // The native binding is an optionalDependency, so "not installed" is a normal + // state on a platform with no prebuild - not an error to log loudly. + return NextResponse.json( + { error: 'Encrypted local storage is unavailable on this platform.', code: 'no-binding' }, + { status: 503 }, + ); + } + return null; +} + +export function replicaErrorResponse(error: unknown, context: string): NextResponse { + if (error instanceof IndexSessionError) { + return NextResponse.json({ error: error.message }, { status: error.status }); + } + if (error instanceof JmapIndexError) { + return NextResponse.json({ error: error.message }, { status: error.status }); + } + if (error instanceof IndexKeyError) { + // `no-secure-storage` is the Linux-without-a-keyring refusal: a real, expected + // outcome with a user-facing explanation, not a server fault. + const status = error.code === 'no-secure-storage' ? 503 : 500; + return NextResponse.json({ error: error.message, code: error.code }, { status }); + } + if (error instanceof ReplicaUnavailableError) { + return NextResponse.json({ error: error.message, code: 'unavailable' }, { status: 503 }); + } + if (error instanceof ReplicaSyncError) { + // A transport failure here means the BACKEND is unreachable, which for a sync + // is an expected outcome rather than a server fault - 503 with the class, so + // the renderer can retry rather than surface an error. + const status = error.cls === 'Auth' ? 401 : error.cls === 'RateLimit' ? 429 : 503; + return NextResponse.json({ error: error.message, code: error.cls }, { status }); + } + logger.error(`offline-replica: ${context} failed`, { + error: error instanceof Error ? error.message : String(error), + }); + return NextResponse.json({ error: `${context} failed` }, { status: 500 }); +} + +export const NO_STORE = { 'Cache-Control': 'no-store' } as const; diff --git a/lib/offline-replica/schema.ts b/lib/offline-replica/schema.ts new file mode 100644 index 00000000..755e0add --- /dev/null +++ b/lib/offline-replica/schema.ts @@ -0,0 +1,178 @@ +// The offline mail replica's schema. +// +// WHY IT LIVES IN THE SAME ENCRYPTED FILE AS THE SEARCH INDEX +// (`lib/mail-index/paths.ts`'s `indexDbPath`), on its own connection: +// +// * One encryption boundary, one key, one keychain entry, one purge. A second +// keystore would double the number of places a key can be mishandled for no +// gain - and `electron/key-service.ts` + the fd-3 channel already work. +// * `sync_state` (cursors, coverage, flags) sits in the SAME FILE as the +// records it describes. That is load-bearing, not tidiness: deleting the file +// removes cursors and records together, so a cursor can never survive a wipe +// and then be advanced over changes that will never be re-delivered. A +// cursor in a sidecar JSON file is exactly the class of bug the mobile +// client's S1 finding is about. +// * The tables are DISJOINT from the index's (`doc`, `doc_fts`), so the two +// subsystems never contend for a row - only, briefly, for SQLite's write +// lock, which `PRAGMA busy_timeout` resolves. Both open with WAL. +// +// The one coupling to accept: `lib/mail-index/store.ts` drops `meta` on an index +// schema bump, which takes `replica_schema_version` with it. The replica reads +// that as "version missing" and purges + re-bootstraps - correct, just wasteful, +// and only on an index schema change. What must NOT happen is records surviving +// while the version row vanishes, which is why the purge below is all-or-nothing +// and includes `replica_sync_state`. +// +// Deliberately NO foreign keys from `replica_email_mailbox.mailbox_id` -> +// `replica_mailbox`, and no cascade from `replica_envelope`. The two change +// streams are not transactionally coupled, so a membership row referencing a +// not-yet-fetched or already-destroyed mailbox is a NORMAL transient state; an +// FK would turn correct behaviour into a constraint violation, and a cascade on +// mailbox deletion would delete mail, violating the deletion-provenance rule +// (only the Email stream may delete an email). + +/** Bumped on any incompatible change. Mismatch = purge + re-bootstrap, never migrate. */ +export const REPLICA_SCHEMA_VERSION = 1; + +export const REPLICA_VERSION_KEY = 'replica_schema_version'; + +export const REPLICA_DDL = ` +CREATE TABLE IF NOT EXISTS replica_mailbox ( + jmap_account_id TEXT NOT NULL, + id TEXT NOT NULL, + name TEXT NOT NULL DEFAULT '', + parent_id TEXT, + role TEXT, + sort_order INTEGER, + total_emails INTEGER, + unread_emails INTEGER, + total_threads INTEGER, + unread_threads INTEGER, + my_rights_json TEXT, + is_subscribed INTEGER, + PRIMARY KEY (jmap_account_id, id) +); + +-- The envelope tier: everything EMAIL_LIST_PROPERTIES carries, so an offline +-- message list renders exactly as an online one does. +CREATE TABLE IF NOT EXISTS replica_envelope ( + jmap_account_id TEXT NOT NULL, + id TEXT NOT NULL, + thread_id TEXT, + received_at TEXT NOT NULL, + size INTEGER, + subject TEXT, + preview TEXT, + from_json TEXT, + to_json TEXT, + cc_json TEXT, + blob_id TEXT, + has_attachment INTEGER NOT NULL DEFAULT 0, + keywords_json TEXT NOT NULL DEFAULT '{}', + -- Owned by the BODY tier. Excluded from the envelope upsert's DO UPDATE SET, + -- or an idempotent page replay would look like "body missing" to the backfill + -- job and re-download every body in the page. + has_body INTEGER NOT NULL DEFAULT 0, + body_bytes INTEGER NOT NULL DEFAULT 0, + cached_at INTEGER NOT NULL, + PRIMARY KEY (jmap_account_id, id) +); +CREATE INDEX IF NOT EXISTS replica_envelope_received + ON replica_envelope(jmap_account_id, received_at DESC); +-- The body-backfill driver: envelopes inside the body window with no body yet. +CREATE INDEX IF NOT EXISTS replica_envelope_nobody + ON replica_envelope(jmap_account_id, has_body, received_at DESC); + +-- Membership is its own table: an email is in many mailboxes, and listing by +-- folder must be an index seek rather than a scan of every cached row. +CREATE TABLE IF NOT EXISTS replica_email_mailbox ( + jmap_account_id TEXT NOT NULL, + email_id TEXT NOT NULL, + mailbox_id TEXT NOT NULL, + PRIMARY KEY (jmap_account_id, email_id, mailbox_id) +); +CREATE INDEX IF NOT EXISTS replica_email_mailbox_by_mailbox + ON replica_email_mailbox(jmap_account_id, mailbox_id); + +-- The body tier. "received_at" is duplicated here on purpose so cap eviction is +-- a single-table ordered scan that cannot be blinded by a missing join. +CREATE TABLE IF NOT EXISTS replica_body ( + jmap_account_id TEXT NOT NULL, + email_id TEXT NOT NULL, + received_at TEXT NOT NULL, + json TEXT NOT NULL, + bytes INTEGER NOT NULL, + PRIMARY KEY (jmap_account_id, email_id) +); +CREATE INDEX IF NOT EXISTS replica_body_received + ON replica_body(jmap_account_id, received_at ASC); + +-- "gave_up" is what makes a body-tier TERMINAL STATE durable, and it is the fix +-- for the worst bug found on the mobile client. Deleting the queue row on +-- give-up was not enough: the backfill job's driver is "envelope with no body", +-- a predicate that CANNOT distinguish "not fetched yet" from "deliberately not +-- kept". So the next pass re-inserted a fresh attempts=0 row and a +-- permanently-failing body was retried five times per cycle, forever. Same +-- shape for a "notFound" body, and worst of all for a body shed by the size cap: +-- shed -> still inside the body window -> re-enqueued -> re-downloaded -> shed +-- again. Unbounded data use with no termination. Keeping the row with a flag is +-- what closes all three. +CREATE TABLE IF NOT EXISTS replica_body_queue ( + jmap_account_id TEXT NOT NULL, + email_id TEXT NOT NULL, + received_at TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at INTEGER, + last_error TEXT, + gave_up INTEGER NOT NULL DEFAULT 0, + gave_up_reason TEXT, + PRIMARY KEY (jmap_account_id, email_id) +); +CREATE INDEX IF NOT EXISTS replica_body_queue_wanted + ON replica_body_queue(jmap_account_id, gave_up, received_at DESC); + +-- Cursors, coverage and flags. Row-per-field, NOT one JSON blob: a blob loaded +-- at cycle start and written at cycle end silently reverts any concurrent write +-- to a different field. On the mobile client that produced an empty record store +-- with a live advanced cursor and resyncRequired reset to false - a permanent +-- silent data gap that is unreachable by design. +CREATE TABLE IF NOT EXISTS replica_sync_state ( + k TEXT PRIMARY KEY, + v TEXT NOT NULL +); +`; + +/** + * Everything a purge removes. `replica_sync_state` is INCLUDED: a record wipe + * that leaves cursors behind is the one state from which no amount of syncing + * recovers, because `/changes` structurally cannot re-deliver mail that already + * existed when the cursor was captured. + */ +export const REPLICA_TABLES: readonly string[] = [ + 'replica_envelope', + 'replica_email_mailbox', + 'replica_body', + 'replica_body_queue', + 'replica_mailbox', + 'replica_sync_state', +]; + +/** Record tables only - used when a reconcile rebuilds without discarding policy. */ +export const REPLICA_RECORD_TABLES: readonly string[] = [ + 'replica_envelope', + 'replica_email_mailbox', + 'replica_body', + 'replica_body_queue', + 'replica_mailbox', +]; + +export const FLAGS_KEY = 'flags'; +export const POLICY_KEY = 'policy'; + +export function cursorStateKey(jmapAccountId: string, type: string): string { + return `cursor:${jmapAccountId}:${type}`; +} + +export function coverageStateKey(jmapAccountId: string): string { + return `coverage:${jmapAccountId}`; +} diff --git a/lib/offline-replica/states.ts b/lib/offline-replica/states.ts new file mode 100644 index 00000000..429b0de2 --- /dev/null +++ b/lib/offline-replica/states.ts @@ -0,0 +1,130 @@ +// Cursor provenance: the type-level machinery that makes "never adopt an +// `Email/get` state as an `Email/changes` cursor" a compile error rather than a +// code-review convention. +// +// This exact bug shipped on the mobile client (its defect D4) and silently +// corrupted sync: `getEmailChanges` returned `null` for ANY error, so a +// transient 503 on `Email/changes` caused an `Email/get` state captured in the +// same cycle to be adopted as the next cursor - fast-forwarding the cursor over +// every change the client had not seen, with no resync. The cost is invisible: +// the store looks healthy and is permanently missing mail. +// +// Two brands, and an ORDERING rule rather than a source rule. "Only a +// `Foo/changes.newState` may ever be a cursor" is tempting but FALSE - bootstrap +// and reconcile legitimately seed from `Foo/get {ids: []}`'s `state`, which +// RFC 8620 s5.1 explicitly permits. A rule the design itself has to violate is a +// rule that gets bypassed at the one call site that matters, so the rule is: +// +// A cursor ADVANCES to a ChangesState from the same (jmapAccountId, type). +// It may be SEEDED from a SnapshotState only inside an EnumerationCommitment +// whose enumeration starts after that snapshot. Nothing else, from anywhere, +// ever becomes a cursor. + +/** Types we hold a `/changes` cursor for. NOT a list of push types. */ +export type CursorType = 'Email' | 'Mailbox'; + +export const CURSOR_TYPES: readonly CursorType[] = ['Email', 'Mailbox']; + +/** From a `Foo/changes` response's `newState`. The only value the delta path may advance to. */ +export type ChangesState = string & { readonly __brand: 'ChangesState' }; + +/** From a `Foo/get` response's `state`. A valid cursor ONLY under the ordering rule above. */ +export type SnapshotState = string & { readonly __brand: 'SnapshotState' }; + +/** + * A JMAP body is parsed JSON, so without a runtime check a `null`, a number or + * an object could be laundered through a cast into something the engine treats + * as a cursor forever. The brand certifies PROVENANCE; this certifies SHAPE. + */ +function certifyStateToken(value: unknown, kind: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError( + `${kind}: expected a non-empty string state token, got ` + + `${value === null ? 'null' : typeof value}`, + ); + } + return value; +} + +/** + * Mint a `ChangesState`. Callable ONLY from the `Foo/changes` response parser in + * `./jmap.ts` - that is the entire point of the brand. There is a test asserting + * no other module casts to these types. + */ +export function asChangesState(newState: unknown): ChangesState { + return certifyStateToken(newState, 'asChangesState') as ChangesState; +} + +/** Mint a `SnapshotState`. Callable ONLY from the `Foo/get` response parser in `./jmap.ts`. */ +export function asSnapshotState(state: unknown): SnapshotState { + return certifyStateToken(state, 'asSnapshotState') as SnapshotState; +} + +/** + * The tag is a REAL, module-private `Symbol()`, deliberately not exported and + * deliberately not `declare const ... : unique symbol`. + * + * - Unexported means no object literal in any other module can produce this + * type, so `mintEnumerationCommitment` is the only constructor. Declaring the + * interface without a symbol tag would let any module falsify it with a + * literal, making the seed path's teeth strictly weaker than + * `advanceCursor`'s - which is the path that needs them most. + * - `declare const x: unique symbol` is TYPE-LEVEL ONLY and emits no runtime + * value, so using it as a computed key throws + * `ReferenceError: x is not defined` the first time the mint runs. That + * mistake is in the superseded design document; it cost the mobile port a + * build failure. A `Symbol()` assigned to a `const` still infers + * `unique symbol`, so unforgeability is identical and no cast is needed. + */ +const enumerationCommitmentTag = Symbol('EnumerationCommitment'); + +/** + * A durable promise to enumerate. Holding one is what entitles a caller to seed + * a cursor from a snapshot state: the snapshot is only a safe cursor because an + * enumeration that starts AFTER it is committed to run. + */ +export interface EnumerationCommitment { + readonly [enumerationCommitmentTag]: true; + readonly jmapAccountId: string; + readonly snapshot: SnapshotState; + /** The retention floor the enumeration is working toward. */ + readonly targetFrom: string; + /** + * The floor PINNED for this enumeration. Equal to `targetFrom` for a + * bootstrap; for a reconcile it is the floor captured at step 0, and the sweep + * deletes only against THIS value, never a `targetFrom` that moved while the + * reconcile was running. + * + * Without the pin: widening retention mid-reconcile (very plausible - the + * reconcile banner is exactly what prompts someone to go change the setting) + * makes the sweep delete against the new wide window while the enumeration + * only covered the old narrow one. Everything in the gap is deleted + * permanently, because `coveredFrom` is then set to the wider floor and + * `/changes` cannot re-deliver pre-existing mail. + */ + readonly sweepFloor: string; + readonly kind: 'bootstrap' | 'reconcile'; +} + +export function mintEnumerationCommitment(args: { + jmapAccountId: string; + snapshot: SnapshotState; + targetFrom: string; + sweepFloor: string; + kind: 'bootstrap' | 'reconcile'; +}): EnumerationCommitment { + return { + [enumerationCommitmentTag]: true, + jmapAccountId: args.jmapAccountId, + snapshot: args.snapshot, + targetFrom: args.targetFrom, + sweepFloor: args.sweepFloor, + kind: args.kind, + }; +} + +export function coveragePhaseForCommitment( + commitment: EnumerationCommitment, +): 'scanning' | 'reconciling' { + return commitment.kind === 'bootstrap' ? 'scanning' : 'reconciling'; +} diff --git a/lib/offline-replica/store.ts b/lib/offline-replica/store.ts new file mode 100644 index 00000000..f95b71d0 --- /dev/null +++ b/lib/offline-replica/store.ts @@ -0,0 +1,997 @@ +// The replica store: the encrypted SQLite file, and every read/write against it. +// +// SYNCHRONOUS ON PURPOSE. `@signalapp/sqlcipher` is a synchronous binding, so +// `transaction()` here takes a synchronous callback and no `await` can ever +// appear inside a `BEGIN ... COMMIT`. That removes an entire hazard class by +// construction: no network call, no timer and no other request can interleave +// with a half-applied transaction. Every JMAP fetch happens OUTSIDE a +// transaction and the results are applied inside one. + +import fs from 'node:fs'; +import path from 'node:path'; +import { loadSqlcipher, type SqlcipherDatabase } from '@/lib/mail-index/binding'; +import { dbSiblings, indexDbPath } from '@/lib/mail-index/paths'; +import { + coverageStateKey, cursorStateKey, FLAGS_KEY, POLICY_KEY, REPLICA_DDL, + REPLICA_RECORD_TABLES, REPLICA_SCHEMA_VERSION, REPLICA_TABLES, REPLICA_VERSION_KEY, +} from './schema'; +import { + coveragePhaseForCommitment, type ChangesState, type CursorType, type EnumerationCommitment, +} from './states'; +import { + defaultFlags, type BodyGiveUpReason, type BodyQueueEntry, type CoverageState, + type EnvelopeRow, type FlagsPatch, type MailboxRow, type ReplicaFlags, type SyncCursor, +} from './types'; + +export class ReplicaUnavailableError extends Error { + constructor(message: string) { + super(message); + this.name = 'ReplicaUnavailableError'; + } +} + +export interface CursorKey { + jmapAccountId: string; + type: CursorType; +} + +/** Retention policy, persisted server-side inside the encrypted store. */ +export interface RetentionPolicy { + envelopeDays: number; + bodyDays: number; + maxBodyMB: number; +} + +export const DEFAULT_POLICY: RetentionPolicy = { + // Envelopes are ~1 KB, so a wide window costs kilobytes per message and means + // a message never falls out of the offline LIST because of a body size cap. + envelopeDays: 180, + bodyDays: 30, + maxBodyMB: 250, +}; + +export const POLICY_LIMITS = { + envelopeDays: { min: 7, max: 3650 }, + bodyDays: { min: 1, max: 3650 }, + maxBodyMB: { min: 16, max: 20_000 }, +} as const; + +export function clampPolicy(raw: Partial | null | undefined): RetentionPolicy { + const pick = ( + value: unknown, + fallback: number, + { min, max }: { min: number; max: number }, + ): number => { + const n = typeof value === 'number' && Number.isFinite(value) ? Math.round(value) : fallback; + return Math.min(Math.max(n, min), max); + }; + const envelopeDays = pick(raw?.envelopeDays, DEFAULT_POLICY.envelopeDays, POLICY_LIMITS.envelopeDays); + const bodyDays = pick(raw?.bodyDays, DEFAULT_POLICY.bodyDays, POLICY_LIMITS.bodyDays); + return { + envelopeDays, + // The body window can never be wider than the envelope window: a body with + // no envelope is an orphan by construction. + bodyDays: Math.min(bodyDays, envelopeDays), + maxBodyMB: pick(raw?.maxBodyMB, DEFAULT_POLICY.maxBodyMB, POLICY_LIMITS.maxBodyMB), + }; +} + +/** + * `PRAGMA cipher_version` must return a non-empty STRING. + * + * Checking the row COUNT instead passes vacuously: a non-SQLCipher binding + * returns ZERO ROWS for this pragma, and `PRAGMA key = ...` is silently accepted + * and does nothing on plain SQLite - no error, a working database, and the mail + * sitting on disk in cleartext. + */ +function assertEncrypted(db: SqlcipherDatabase, dbPath: string): void { + const rows = db.pragma('cipher_version'); + const value = + Array.isArray(rows) && rows.length > 0 && rows[0] && typeof rows[0] === 'object' + ? (rows[0] as Record).cipher_version + : undefined; + if (typeof value !== 'string' || value.trim().length === 0) { + db.close(); + throw new ReplicaUnavailableError( + `Refusing to use ${path.basename(dbPath)}: the SQLite binding reports no SQLCipher ` + + `support (PRAGMA cipher_version returned ${JSON.stringify(rows)}), so the offline ` + + `replica would be written in cleartext.`, + ); + } +} + +function num(v: unknown): number | null { + return typeof v === 'number' && Number.isFinite(v) ? v : null; +} + +function str(v: unknown): string | null { + return typeof v === 'string' ? v : null; +} + +export interface OpenReplicaOptions { + storeDir: string; + accountId: string; + /** Raw 32-byte key, from the main process's key service. */ + key: Buffer; +} + +export class ReplicaStore { + private constructor( + private readonly db: SqlcipherDatabase, + readonly dbPath: string, + ) {} + + static open({ storeDir, accountId, key }: OpenReplicaOptions): ReplicaStore { + const Database = loadSqlcipher(); + if (!Database) { + throw new ReplicaUnavailableError( + '@signalapp/sqlcipher is not installed for this platform (it is an optional dependency).', + ); + } + if (key.length !== 32) { + throw new ReplicaUnavailableError(`Replica key must be 32 bytes, got ${key.length}.`); + } + + // The SAME file as the search index. See schema.ts for why. + const dbPath = indexDbPath(storeDir, accountId); + fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: 0o700 }); + + const connect = (): SqlcipherDatabase => { + const db = new Database(dbPath); + // The key pragma must be the FIRST statement on the connection. Hex form + // means SQLCipher uses these 32 bytes as the raw key with no KDF. + db.pragma(`key = "x'${key.toString('hex')}'"`); + assertEncrypted(db, dbPath); + return db; + }; + + let db = connect(); + let version: number | null; + try { + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = NORMAL'); + // The index and the replica are two connections to one file. WAL lets a + // writer and readers coexist, but two WRITERS get SQLITE_BUSY immediately + // without this - and both are driven by the same renderer push handler, so + // they genuinely do overlap. + db.pragma('busy_timeout = 8000'); + version = readVersion(db); + } catch { + // A wrong key surfaces here, not at open: SQLCipher reads the header + // lazily. The replica is derived data, so there is nothing to recover and + // never anything to prompt the user for. + db.close(); + for (const f of dbSiblings(dbPath)) { + try { fs.rmSync(f, { force: true }); } catch { /* best effort */ } + } + db = connect(); + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = NORMAL'); + db.pragma('busy_timeout = 8000'); + version = null; + } + + if (version !== null && version !== REPLICA_SCHEMA_VERSION) version = null; + + if (version === null) { + // ALL-OR-NOTHING. Records must never survive while the version row is + // gone: a cursor that outlives its records is the one state no amount of + // syncing repairs, because `/changes` cannot re-deliver mail that already + // existed when the cursor was captured. + db.exec('BEGIN'); + try { + for (const table of REPLICA_TABLES) db.exec(`DROP TABLE IF EXISTS ${table}`); + db.exec(REPLICA_DDL); + db.exec('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL)'); + db.prepare('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)').run([ + REPLICA_VERSION_KEY, + String(REPLICA_SCHEMA_VERSION), + ]); + db.exec('COMMIT'); + } catch (error) { + db.exec('ROLLBACK'); + db.close(); + throw error; + } + } else { + // The tables exist per the version row, but `CREATE TABLE IF NOT EXISTS` + // is cheap and covers a partially-created file from an interrupted open. + db.exec(REPLICA_DDL); + } + + return new ReplicaStore(db, dbPath); + } + + close(): void { + try { this.db.close(); } catch { /* already closed */ } + } + + /** + * One SQLite transaction. The callback is SYNCHRONOUS, so nothing can + * interleave and no `await` can sit inside `BEGIN ... COMMIT`. + */ + transaction(fn: () => T): T { + this.db.exec('BEGIN'); + try { + const out = fn(); + this.db.exec('COMMIT'); + return out; + } catch (error) { + try { this.db.exec('ROLLBACK'); } catch { /* the commit may have failed */ } + throw error; + } + } + + // ── raw sync_state access ──────────────────────────────────────────────── + + private readState(k: string): T | null { + const row = this.db.prepare('SELECT v FROM replica_sync_state WHERE k = ?').get([k]); + if (!row || typeof row.v !== 'string') return null; + try { + return JSON.parse(row.v) as T; + } catch { + // A corrupt state blob is a resync signal, not something to guess at. + return null; + } + } + + private writeState(k: string, value: unknown): void { + this.db + .prepare('INSERT INTO replica_sync_state (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v') + .run([k, JSON.stringify(value)]); + } + + // ── policy ─────────────────────────────────────────────────────────────── + + getPolicy(): RetentionPolicy { + return clampPolicy(this.readState>(POLICY_KEY)); + } + + setPolicy(policy: RetentionPolicy): void { + this.writeState(POLICY_KEY, clampPolicy(policy)); + } + + // ── flags ──────────────────────────────────────────────────────────────── + + getFlags(now: number): ReplicaFlags { + return this.readState(FLAGS_KEY) ?? defaultFlags(now); + } + + patchFlags(now: number, patch: FlagsPatch): void { + const current = this.getFlags(now); + this.writeState(FLAGS_KEY, { ...current, ...patch }); + } + + // ── cursors ────────────────────────────────────────────────────────────── + + getCursor(key: CursorKey): SyncCursor | null { + return this.readState(cursorStateKey(key.jmapAccountId, key.type)); + } + + /** + * The delta path's ONLY cursor write. The signature is what makes the mobile + * client's D4 a compile error here: a `SnapshotState` cannot be passed. + * + * Throws when the cursor does not exist. A cursor is born from `seedCursor` + * and nowhere else; creating one here would be a silent cursor-from-nowhere, + * which is the exact class of bug the branded types exist to prevent. + */ + advanceCursor(key: CursorKey, next: ChangesState): void { + const k = cursorStateKey(key.jmapAccountId, key.type); + const current = this.readState(k); + if (!current) { + throw new Error( + `advanceCursor: no cursor for ${key.type}/${key.jmapAccountId}; seed it first`, + ); + } + this.writeState(k, { ...current, state: next, updatedAt: Date.now() } satisfies SyncCursor); + } + + /** + * Bootstrap / reconcile only. Writes the snapshot state AND the `CoverageState` + * it justifies in the SAME transaction, so a seed is never durable without the + * durable commitment to enumerate that justifies it. + * + * Call inside `transaction()`. + */ + seedCursor(key: CursorKey, commitment: EnumerationCommitment, now: number): void { + if (commitment.jmapAccountId !== key.jmapAccountId) { + throw new Error('seedCursor: commitment is for a different JMAP account'); + } + const seeded: SyncCursor = { + type: key.type, + jmapAccountId: key.jmapAccountId, + state: commitment.snapshot, + drainPending: false, + consecutiveFailures: 0, + maxChangesRung: 0, + updatedAt: now, + }; + this.writeState(cursorStateKey(key.jmapAccountId, key.type), seeded); + + const existing = this.getCoverage(key.jmapAccountId); + const next: CoverageState = { + jmapAccountId: key.jmapAccountId, + // Records stay readable during a reconcile, so what was already covered + // stays claimed until the reconcile finishes and sets the pinned floor. + coveredFrom: existing?.coveredFrom ?? null, + scanCursor: null, + targetFrom: commitment.targetFrom, + sweepFloor: commitment.sweepFloor, + deferredTargetFrom: undefined, + gapMarkers: existing?.gapMarkers, + phase: coveragePhaseForCommitment(commitment), + seen: 0, + consecutiveFailures: 0, + updatedAt: now, + }; + this.writeState(coverageStateKey(key.jmapAccountId), next); + } + + /** Field-level patch. `state` is deliberately NOT patchable - see advance/seed. */ + patchCursor( + key: CursorKey, + patch: Partial>, + ): void { + const k = cursorStateKey(key.jmapAccountId, key.type); + const current = this.readState(k); + if (!current) return; + this.writeState(k, { ...current, ...patch, updatedAt: Date.now() }); + } + + // ── coverage ───────────────────────────────────────────────────────────── + + getCoverage(jmapAccountId: string): CoverageState | null { + return this.readState(coverageStateKey(jmapAccountId)); + } + + patchCoverage(jmapAccountId: string, patch: Partial): void { + const k = coverageStateKey(jmapAccountId); + const current = this.readState(k); + if (!current) return; + this.writeState(k, { ...current, ...patch, updatedAt: Date.now() }); + } + + // ── mailboxes ──────────────────────────────────────────────────────────── + + upsertMailboxes(rows: readonly MailboxRow[]): number { + if (rows.length === 0) return 0; + const stmt = this.db.prepare(` + INSERT INTO replica_mailbox (jmap_account_id, id, name, parent_id, role, sort_order, + total_emails, unread_emails, total_threads, unread_threads, my_rights_json, is_subscribed) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(jmap_account_id, id) DO UPDATE SET + name = excluded.name, parent_id = excluded.parent_id, role = excluded.role, + sort_order = excluded.sort_order, total_emails = excluded.total_emails, + unread_emails = excluded.unread_emails, total_threads = excluded.total_threads, + unread_threads = excluded.unread_threads, my_rights_json = excluded.my_rights_json, + is_subscribed = excluded.is_subscribed + `); + for (const r of rows) { + stmt.run([ + r.jmapAccountId, r.id, r.name, r.parentId, r.role, r.sortOrder, + r.totalEmails, r.unreadEmails, r.totalThreads, r.unreadThreads, + r.myRightsJson, r.isSubscribed ? 1 : 0, + ]); + } + return rows.length; + } + + /** + * Patches ONLY the four count columns. + * + * `Mailbox/changes` reports `updatedProperties` as an upper bound of what may + * have changed (RFC 8621 s2.2), and counts move on every delivery and every + * read. On a busy account this is the difference between patching four + * integers and re-fetching every folder object. + */ + patchMailboxCounts( + jmapAccountId: string, + id: string, + counts: { + totalEmails?: number | null; unreadEmails?: number | null; + totalThreads?: number | null; unreadThreads?: number | null; + }, + ): void { + const sets: string[] = []; + const params: unknown[] = []; + for (const [column, value] of [ + ['total_emails', counts.totalEmails], ['unread_emails', counts.unreadEmails], + ['total_threads', counts.totalThreads], ['unread_threads', counts.unreadThreads], + ] as const) { + if (value !== undefined) { sets.push(`${column} = ?`); params.push(value); } + } + if (sets.length === 0) return; + this.db + .prepare(`UPDATE replica_mailbox SET ${sets.join(', ')} WHERE jmap_account_id = ? AND id = ?`) + .run([...params, jmapAccountId, id]); + } + + /** + * Deletes the mailbox row ONLY. Never touches email records. + * + * Deletion provenance: if the server destroyed the messages too, + * `Email/changes` reports them `destroyed`; if it moved them, their + * `mailboxIds` update arrives as `updated`. Truth arrives on the Email stream + * either way. Inferring deletion from a mailbox disappearing is how a client + * loses mail the server still has. + */ + deleteMailboxes(jmapAccountId: string, ids: readonly string[]): number { + if (ids.length === 0) return 0; + const stmt = this.db.prepare('DELETE FROM replica_mailbox WHERE jmap_account_id = ? AND id = ?'); + let n = 0; + for (const id of ids) n += stmt.run([jmapAccountId, id]).changes; + return n; + } + + listMailboxes(jmapAccountId: string): MailboxRow[] { + return this.db + .prepare('SELECT * FROM replica_mailbox WHERE jmap_account_id = ? ORDER BY sort_order ASC, name ASC') + .all([jmapAccountId]) + .map((r) => ({ + jmapAccountId: String(r.jmap_account_id), + id: String(r.id), + name: String(r.name ?? ''), + parentId: str(r.parent_id), + role: str(r.role), + sortOrder: num(r.sort_order), + totalEmails: num(r.total_emails), + unreadEmails: num(r.unread_emails), + totalThreads: num(r.total_threads), + unreadThreads: num(r.unread_threads), + myRightsJson: str(r.my_rights_json), + isSubscribed: r.is_subscribed !== 0, + })); + } + + // ── envelopes ──────────────────────────────────────────────────────────── + + /** + * Upserts envelopes and replaces their membership rows. + * + * `has_body` / `body_bytes` are DELIBERATELY absent from `DO UPDATE SET`: they + * belong to the body tier, and resetting them on an idempotent page replay + * would look like "body missing" to the backfill job and re-download every + * body in the page. + * + * `cachedAt` is a parameter rather than `Date.now()` because a reconcile must + * stamp with its PINNED value - see `CoverageState.reconcileStampedAt`. + */ + upsertEnvelopes(rows: readonly EnvelopeRow[], cachedAt: number): number { + if (rows.length === 0) return 0; + const upsert = this.db.prepare(` + INSERT INTO replica_envelope (jmap_account_id, id, thread_id, received_at, size, subject, + preview, from_json, to_json, cc_json, blob_id, has_attachment, keywords_json, + has_body, body_bytes, cached_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?) + ON CONFLICT(jmap_account_id, id) DO UPDATE SET + thread_id = excluded.thread_id, received_at = excluded.received_at, + size = excluded.size, subject = excluded.subject, preview = excluded.preview, + from_json = excluded.from_json, to_json = excluded.to_json, cc_json = excluded.cc_json, + blob_id = excluded.blob_id, has_attachment = excluded.has_attachment, + keywords_json = excluded.keywords_json, cached_at = excluded.cached_at + `); + const clearMembership = this.db.prepare( + 'DELETE FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?', + ); + const addMembership = this.db.prepare( + 'INSERT OR IGNORE INTO replica_email_mailbox (jmap_account_id, email_id, mailbox_id) VALUES (?, ?, ?)', + ); + for (const r of rows) { + upsert.run([ + r.jmapAccountId, r.id, r.threadId, r.receivedAt, r.size, r.subject, r.preview, + r.fromJson, r.toJson, r.ccJson, r.blobId, r.hasAttachment ? 1 : 0, r.keywordsJson, + cachedAt, + ]); + clearMembership.run([r.jmapAccountId, r.id]); + for (const mailboxId of r.mailboxIds) addMembership.run([r.jmapAccountId, r.id, mailboxId]); + } + return rows.length; + } + + /** + * Patches the only two MUTABLE Email properties (RFC 8621 s4.1): `keywords` + * and `mailboxIds`. Everything else - body, attachments, headers, receivedAt, + * size, threadId, preview, subject, addresses - is immutable for the lifetime + * of the id, which is why an `updated` id never needs a body re-fetch. + * + * No-ops for an id we do not hold, and only touches membership when the + * envelope row actually existed, or we would leave membership rows for a + * record we do not have. + */ + patchEnvelopeMutable( + jmapAccountId: string, + id: string, + patch: { keywordsJson: string; mailboxIds: string[] }, + ): boolean { + const res = this.db + .prepare('UPDATE replica_envelope SET keywords_json = ? WHERE jmap_account_id = ? AND id = ?') + .run([patch.keywordsJson, jmapAccountId, id]); + if (res.changes === 0) return false; + this.db + .prepare('DELETE FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?') + .run([jmapAccountId, id]); + const add = this.db.prepare( + 'INSERT OR IGNORE INTO replica_email_mailbox (jmap_account_id, email_id, mailbox_id) VALUES (?, ?, ?)', + ); + for (const mailboxId of patch.mailboxIds) add.run([jmapAccountId, id, mailboxId]); + return true; + } + + /** Bulk presence test, so the delta path can filter `updated` ids BEFORE fetching. */ + whichEnvelopesExist(jmapAccountId: string, ids: readonly string[]): Set { + if (ids.length === 0) return new Set(); + const out = new Set(); + const stmt = this.db.prepare( + 'SELECT id FROM replica_envelope WHERE jmap_account_id = ? AND id = ?', + ); + for (const id of ids) { + if (stmt.get([jmapAccountId, id])) out.add(id); + } + return out; + } + + /** Deletes an email everywhere: envelope, body, membership and any queue row. */ + deleteEmails(jmapAccountId: string, ids: readonly string[]): number { + if (ids.length === 0) return 0; + const statements = [ + this.db.prepare('DELETE FROM replica_body WHERE jmap_account_id = ? AND email_id = ?'), + this.db.prepare('DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND email_id = ?'), + this.db.prepare('DELETE FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?'), + ]; + const deleteEnvelope = this.db.prepare( + 'DELETE FROM replica_envelope WHERE jmap_account_id = ? AND id = ?', + ); + let n = 0; + for (const id of ids) { + for (const s of statements) s.run([jmapAccountId, id]); + n += deleteEnvelope.run([jmapAccountId, id]).changes; + } + return n; + } + + /** Retention eviction: everything strictly older than the floor. */ + evictEnvelopesBelow(jmapAccountId: string, isoFloor: string): number { + const ids = this.db + .prepare('SELECT id FROM replica_envelope WHERE jmap_account_id = ? AND received_at < ?') + .all([jmapAccountId, isoFloor]) + .map((r) => String(r.id)); + return this.deleteEmails(jmapAccountId, ids); + } + + /** + * The reconcile sweep. Two clauses, and it REFUSES to run without a pinned + * stamp rather than deleting unverified records. + */ + sweep(jmapAccountId: string, sweepFloor: string, reconcileStampedAt: number | undefined): number { + if (reconcileStampedAt === undefined) { + throw new Error('sweep: no reconcileStampedAt pinned; refusing to delete unverified records'); + } + const notReSeen = this.db + .prepare(` + SELECT id FROM replica_envelope + WHERE jmap_account_id = ? AND received_at >= ? AND cached_at < ? + `) + .all([jmapAccountId, sweepFloor, reconcileStampedAt]) + .map((r) => String(r.id)); + // Records older than the pinned floor cannot be verified by an enumeration + // that only covers the window, so they go rather than being kept on faith. + // Normally retention has already evicted them. + const unverifiable = this.db + .prepare('SELECT id FROM replica_envelope WHERE jmap_account_id = ? AND received_at < ?') + .all([jmapAccountId, sweepFloor]) + .map((r) => String(r.id)); + return this.deleteEmails(jmapAccountId, [...new Set([...notReSeen, ...unverifiable])]); + } + + /** + * The reconcile stamp must be derived from the DATA, not the clock: + * `max(now, maxCachedAt + 1)`. With a frozen or coarse clock, + * `cached_at < stamp` matches nothing and the sweep silently deletes nothing. + */ + maxEnvelopeCachedAt(jmapAccountId: string): number { + const row = this.db + .prepare('SELECT MAX(cached_at) AS m FROM replica_envelope WHERE jmap_account_id = ?') + .get([jmapAccountId]); + return num(row?.m) ?? 0; + } + + countEnvelopes(jmapAccountId: string): number { + const row = this.db + .prepare('SELECT COUNT(*) AS n FROM replica_envelope WHERE jmap_account_id = ?') + .get([jmapAccountId]); + return num(row?.n) ?? 0; + } + + /** Envelopes inside the body window with no body yet - the backfill driver. */ + envelopesWithoutBody( + jmapAccountId: string, + receivedAfter: string, + limit: number, + ): Array<{ id: string; receivedAt: string; size: number }> { + return this.db + .prepare(` + SELECT id, received_at, size FROM replica_envelope + WHERE jmap_account_id = ? AND has_body = 0 AND received_at >= ? + ORDER BY received_at DESC LIMIT ? + `) + .all([jmapAccountId, receivedAfter, limit]) + .map((r) => ({ + id: String(r.id), + receivedAt: String(r.received_at), + size: num(r.size) ?? 0, + })); + } + + // ── bodies ─────────────────────────────────────────────────────────────── + + /** + * Writes a body ONLY if its envelope still exists, and returns whether it did. + * + * Without the condition, a body fetched moments before its envelope was + * destroyed in the same cycle lands as an orphan. This is also exactly why + * "run bodies in parallel, it's separate state" is forbidden. + */ + putBodyIfEnvelopeExists(jmapAccountId: string, emailId: string, json: string): boolean { + const envelope = this.db + .prepare('SELECT received_at FROM replica_envelope WHERE jmap_account_id = ? AND id = ?') + .get([jmapAccountId, emailId]); + const receivedAt = str(envelope?.received_at); + if (receivedAt === null) return false; + const bytes = Buffer.byteLength(json, 'utf8'); + this.db + .prepare(` + INSERT INTO replica_body (jmap_account_id, email_id, received_at, json, bytes) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(jmap_account_id, email_id) DO UPDATE SET + received_at = excluded.received_at, json = excluded.json, bytes = excluded.bytes + `) + .run([jmapAccountId, emailId, receivedAt, json, bytes]); + this.db + .prepare('UPDATE replica_envelope SET has_body = 1, body_bytes = ? WHERE jmap_account_id = ? AND id = ?') + .run([bytes, jmapAccountId, emailId]); + return true; + } + + getBody(jmapAccountId: string, emailId: string): string | null { + const row = this.db + .prepare('SELECT json FROM replica_body WHERE jmap_account_id = ? AND email_id = ?') + .get([jmapAccountId, emailId]); + return str(row?.json); + } + + deleteBodies(jmapAccountId: string, emailIds: readonly string[]): number { + if (emailIds.length === 0) return 0; + const del = this.db.prepare( + 'DELETE FROM replica_body WHERE jmap_account_id = ? AND email_id = ?', + ); + const clearFlag = this.db.prepare( + 'UPDATE replica_envelope SET has_body = 0, body_bytes = 0 WHERE jmap_account_id = ? AND id = ?', + ); + let n = 0; + for (const id of emailIds) { + n += del.run([jmapAccountId, id]).changes; + clearFlag.run([jmapAccountId, id]); + } + return n; + } + + bodyBytesTotal(jmapAccountId: string): number { + const row = this.db + .prepare('SELECT COALESCE(SUM(bytes), 0) AS n FROM replica_body WHERE jmap_account_id = ?') + .get([jmapAccountId]); + return num(row?.n) ?? 0; + } + + countBodies(jmapAccountId: string): number { + const row = this.db + .prepare('SELECT COUNT(*) AS n FROM replica_body WHERE jmap_account_id = ?') + .get([jmapAccountId]); + return num(row?.n) ?? 0; + } + + /** Oldest bodies first - the cap-eviction order. Envelopes always survive. */ + oldestBodies(jmapAccountId: string, limit: number): Array<{ emailId: string; bytes: number }> { + return this.db + .prepare(` + SELECT email_id, bytes FROM replica_body WHERE jmap_account_id = ? + ORDER BY received_at ASC, email_id ASC LIMIT ? + `) + .all([jmapAccountId, limit]) + .map((r) => ({ emailId: String(r.email_id), bytes: num(r.bytes) ?? 0 })); + } + + /** Bodies below the body-retention floor. */ + bodiesBelow(jmapAccountId: string, isoFloor: string, limit: number): string[] { + return this.db + .prepare(` + SELECT email_id FROM replica_body + WHERE jmap_account_id = ? AND received_at < ? ORDER BY received_at ASC LIMIT ? + `) + .all([jmapAccountId, isoFloor, limit]) + .map((r) => String(r.email_id)); + } + + /** Bodies whose envelope is gone. Invisible to cap eviction, which walks the body table. */ + orphanBodies(jmapAccountId: string, limit: number): string[] { + return this.db + .prepare(` + SELECT b.email_id FROM replica_body b + LEFT JOIN replica_envelope e + ON e.jmap_account_id = b.jmap_account_id AND e.id = b.email_id + WHERE b.jmap_account_id = ? AND e.id IS NULL LIMIT ? + `) + .all([jmapAccountId, limit]) + .map((r) => String(r.email_id)); + } + + // ── body queue ─────────────────────────────────────────────────────────── + + /** + * Insert-or-ignore. NEVER resets `attempts` on an existing row, and never + * revives a `gave_up` row. + * + * Returns the number of rows ACTUALLY INSERTED. The distinction matters: the + * caller reports this as progress, and reporting attempted-rather-than-inserted + * made the mobile engine believe there was unfinished work on every cycle for + * as long as any envelope lacked a body - an endless chain of cycles seconds + * apart, changing nothing. + */ + enqueueBodies(entries: readonly BodyQueueEntry[]): number { + if (entries.length === 0) return 0; + const stmt = this.db.prepare(` + INSERT OR IGNORE INTO replica_body_queue + (jmap_account_id, email_id, received_at, attempts, next_attempt_at, last_error, gave_up, gave_up_reason) + VALUES (?, ?, ?, ?, ?, ?, 0, NULL) + `); + let inserted = 0; + for (const e of entries) { + inserted += stmt.run([ + e.jmapAccountId, e.emailId, e.receivedAt, e.attempts, e.nextAttemptAt ?? null, + e.lastError ?? null, + ]).changes; + } + return inserted; + } + + /** Rows still WANTED: not given up, and past any backoff. Newest first. */ + takeBodyQueue(jmapAccountId: string, limit: number, now: number): BodyQueueEntry[] { + return this.db + .prepare(` + SELECT * FROM replica_body_queue + WHERE jmap_account_id = ? AND gave_up = 0 + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + ORDER BY received_at DESC LIMIT ? + `) + .all([jmapAccountId, now, limit]) + .map((r) => ({ + emailId: String(r.email_id), + jmapAccountId: String(r.jmap_account_id), + receivedAt: String(r.received_at), + attempts: num(r.attempts) ?? 0, + lastError: str(r.last_error) ?? undefined, + nextAttemptAt: num(r.next_attempt_at) ?? undefined, + gaveUp: r.gave_up !== 0, + gaveUpReason: (str(r.gave_up_reason) as BodyGiveUpReason | null) ?? undefined, + })); + } + + dequeueBodies(jmapAccountId: string, emailIds: readonly string[]): number { + if (emailIds.length === 0) return 0; + const stmt = this.db.prepare( + 'DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND email_id = ?', + ); + let n = 0; + for (const id of emailIds) n += stmt.run([jmapAccountId, id]).changes; + return n; + } + + bumpBodyAttempt( + jmapAccountId: string, + emailId: string, + nextAttemptAt: number, + lastError: string, + ): void { + this.db + .prepare(` + UPDATE replica_body_queue SET attempts = attempts + 1, next_attempt_at = ?, last_error = ? + WHERE jmap_account_id = ? AND email_id = ? + `) + .run([nextAttemptAt, lastError.slice(0, 400), jmapAccountId, emailId]); + } + + /** Records a durable terminal state INSTEAD of deleting the row. */ + markBodyGaveUp( + jmapAccountId: string, + entries: ReadonlyArray<{ emailId: string; receivedAt: string; reason: BodyGiveUpReason; lastError?: string }>, + ): void { + if (entries.length === 0) return; + // A cap-shed body may have no queue row at all (it was fetched and stored + // successfully, then evicted), so this must be an upsert rather than an + // update - otherwise the mark is silently dropped and the shed/re-download + // loop stays open. + const stmt = this.db.prepare(` + INSERT INTO replica_body_queue + (jmap_account_id, email_id, received_at, attempts, next_attempt_at, last_error, gave_up, gave_up_reason) + VALUES (?, ?, ?, 0, NULL, ?, 1, ?) + ON CONFLICT(jmap_account_id, email_id) DO UPDATE SET + gave_up = 1, gave_up_reason = excluded.gave_up_reason, + last_error = excluded.last_error, next_attempt_at = NULL + `); + for (const e of entries) { + stmt.run([jmapAccountId, e.emailId, e.receivedAt, e.lastError?.slice(0, 400) ?? null, e.reason]); + } + } + + listBodyGiveUps(jmapAccountId: string, limit: number): string[] { + return this.db + .prepare('SELECT email_id FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1 LIMIT ?') + .all([jmapAccountId, limit]) + .map((r) => String(r.email_id)); + } + + /** + * DELETES give-up rows rather than un-flagging them, so a cleared give-up + * looks like "never queued" and the backfill pass re-enqueues it with a clean + * attempt count. + * + * Called unconditionally by a completed reconcile: a give-up recorded during + * whatever went wrong must not outlive it, or a transient outage would + * permanently deny a body with no path back. + */ + clearBodyGiveUps(jmapAccountId: string, reason?: BodyGiveUpReason): number { + if (reason) { + return this.db + .prepare('DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1 AND gave_up_reason = ?') + .run([jmapAccountId, reason]).changes; + } + return this.db + .prepare('DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1') + .run([jmapAccountId]).changes; + } + + countWantedBodies(jmapAccountId: string, now: number): number { + const row = this.db + .prepare(` + SELECT COUNT(*) AS n FROM replica_body_queue + WHERE jmap_account_id = ? AND gave_up = 0 + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + `) + .get([jmapAccountId, now]); + return num(row?.n) ?? 0; + } + + // ── purge ──────────────────────────────────────────────────────────────── + + /** Wipes records AND the body queue. Leaves `replica_sync_state` (policy, cursors). */ + clearRecords(): void { + for (const table of REPLICA_RECORD_TABLES) this.db.exec(`DELETE FROM ${table}`); + } + + /** Everything, cursors included. The only safe pairing with a record wipe. */ + purgeAll(): void { + for (const table of REPLICA_TABLES) this.db.exec(`DELETE FROM ${table}`); + } + + // ── read path ──────────────────────────────────────────────────────────── + + /** Envelope page for a mailbox, newest first. `mailboxId === null` = all mail. */ + listEnvelopes( + jmapAccountId: string, + mailboxId: string | null, + limit: number, + offset: number, + ): { rows: Array>; total: number } { + if (mailboxId === null) { + const total = + num( + this.db + .prepare('SELECT COUNT(*) AS n FROM replica_envelope WHERE jmap_account_id = ?') + .get([jmapAccountId])?.n, + ) ?? 0; + const rows = this.db + .prepare(` + SELECT * FROM replica_envelope WHERE jmap_account_id = ? + ORDER BY received_at DESC, id DESC LIMIT ? OFFSET ? + `) + .all([jmapAccountId, limit, offset]); + return { rows, total }; + } + const total = + num( + this.db + .prepare('SELECT COUNT(*) AS n FROM replica_email_mailbox WHERE jmap_account_id = ? AND mailbox_id = ?') + .get([jmapAccountId, mailboxId])?.n, + ) ?? 0; + const rows = this.db + .prepare(` + SELECT e.* FROM replica_envelope e + JOIN replica_email_mailbox m + ON m.jmap_account_id = e.jmap_account_id AND m.email_id = e.id + WHERE e.jmap_account_id = ? AND m.mailbox_id = ? + ORDER BY e.received_at DESC, e.id DESC LIMIT ? OFFSET ? + `) + .all([jmapAccountId, mailboxId, limit, offset]); + return { rows, total }; + } + + getEnvelopeRaw(jmapAccountId: string, id: string): Record | null { + const row = this.db + .prepare('SELECT * FROM replica_envelope WHERE jmap_account_id = ? AND id = ?') + .get([jmapAccountId, id]); + return row ?? null; + } + + mailboxIdsFor(jmapAccountId: string, emailId: string): string[] { + return this.db + .prepare('SELECT mailbox_id FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?') + .all([jmapAccountId, emailId]) + .map((r) => String(r.mailbox_id)); + } + + /** Size + freshness, for the Settings surface. */ + stats(jmapAccountId: string): { + mailboxes: number; + envelopes: number; + bodies: number; + bodyBytes: number; + wantedBodies: number; + giveUps: number; + newest: string | null; + oldest: string | null; + fileBytes: number; + } { + const mailboxes = + num( + this.db + .prepare('SELECT COUNT(*) AS n FROM replica_mailbox WHERE jmap_account_id = ?') + .get([jmapAccountId])?.n, + ) ?? 0; + const range = this.db + .prepare('SELECT MIN(received_at) AS lo, MAX(received_at) AS hi FROM replica_envelope WHERE jmap_account_id = ?') + .get([jmapAccountId]); + let fileBytes = 0; + for (const f of dbSiblings(this.dbPath)) { + try { fileBytes += fs.statSync(f).size; } catch { /* absent sibling */ } + } + return { + mailboxes, + envelopes: this.countEnvelopes(jmapAccountId), + bodies: this.countBodies(jmapAccountId), + bodyBytes: this.bodyBytesTotal(jmapAccountId), + wantedBodies: this.countWantedBodies(jmapAccountId, Date.now()), + giveUps: + num( + this.db + .prepare('SELECT COUNT(*) AS n FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1') + .get([jmapAccountId])?.n, + ) ?? 0, + newest: str(range?.hi), + oldest: str(range?.lo), + fileBytes, + }; + } + + /** Every JMAP account id with rows, so the read path can find them without a session. */ + knownJmapAccountIds(): string[] { + const ids = new Set(); + for (const table of ['replica_envelope', 'replica_mailbox'] as const) { + for (const r of this.db.prepare(`SELECT DISTINCT jmap_account_id FROM ${table}`).all()) { + if (typeof r.jmap_account_id === 'string') ids.add(r.jmap_account_id); + } + } + return [...ids]; + } +} + +function readVersion(db: SqlcipherDatabase): number | null { + try { + const row = db.prepare('SELECT v FROM meta WHERE k = ?').get([REPLICA_VERSION_KEY]); + if (!row || row.v === undefined) return null; + const n = Number(row.v); + return Number.isFinite(n) ? n : null; + } catch { + // `meta` doesn't exist yet - a fresh file. + return null; + } +} diff --git a/lib/offline-replica/sync.ts b/lib/offline-replica/sync.ts new file mode 100644 index 00000000..7d2e7bac --- /dev/null +++ b/lib/offline-replica/sync.ts @@ -0,0 +1,1122 @@ +// One sync cycle. Request-scoped, bounded, single-flighted. +// +// PROCESS ARCHITECTURE - the decision that keeps most of the design review's +// findings out of scope. There is NO persistent background worker. A cycle is +// ordinary work inside an API route, using the request's own encrypted +// `jmap_stalwart_ctx` cookie via `lib/stalwart/credentials.ts`, exactly as the +// search index already does. That is deliberate, because the adversarial review +// of the original full-replica design killed four of its findings by removing the +// worker rather than fixing them: +// +// C2 - "credentials are request-scoped, so no persistent worker can hold them". +// Still true, and still fine, because there is no worker. A cycle only ever +// reads an ALREADY-MINTED Authorization header off the request. +// C3 - the OAuth-refresh mitigation being itself the bug. Avoided by +// construction: nothing here touches the refresh-token cookie, so it cannot +// rotate a token into a response nobody reads and log the user out. +// C4 - a shared `registry.json` breaking the epoch fencing token. No registry, +// no epochs: single-flight per account inside one process, and every piece +// of state lives in the SQLite file under a real transaction. +// H1 - "a server-side engine cannot read a renderer-only setting". The renderer +// decides when to sync, so nothing materialises for an account that never +// opted in. The retention POLICY does need to be durable server-side, so it +// lives inside the encrypted store (written through PUT /api/offline/status), +// not in renderer localStorage. +// H4 - unbounded concurrent multi-account sync starving foreground activity. +// One request, one account, one cycle, hard budgets below. +// +// H2 (key handoff via process environment) was already fixed by what shipped: the +// key crosses on an inherited file descriptor and is zeroed after each job. +// C1 (the native dependency breaking Alpine `docker build`s) is likewise already +// fixed and this file adds no new dependency. +// +// H3 is the one finding that genuinely COMES BACK. The webmail does local delta +// arithmetic on mailbox unread counts and totals for mark-read/move/delete, and a +// read-only offline cache sitting underneath that arithmetic needs a coherence +// story. The story is in `read.ts`: the replica is a FALLBACK, never a cache in +// front of the server. It is consulted only after a read has actually failed at +// the transport level, so an online session never sees a replica count. +// +// JOB ORDER within a cycle is A1 (Mailbox/changes) -> A2 (Email/changes) -> +// B (coverage enumeration) -> C1 (body queue drain) -> C2 (body backfill) -> +// retention. The three machines are logically independent but OPERATIONALLY +// SERIALISED. Either job order is safe; CONCURRENCY is not - coverage's +// query-then-apply pair interleaved with the delta path's apply resurrects a +// destroyed message as a zombie no future `/changes` page will ever re-report. +// "Run bodies in parallel, it's separate state" is forbidden for the same reason. + +import type { Email, Mailbox } from '@/lib/jmap/types'; +import { logger } from '@/lib/logger'; +import type { JmapSessionInfo } from '@/lib/mail-index/jmap'; +import { + madeForwardProgress, planEmailFetches, planMailboxFetches, pageIsEmpty, advanceOneMs, +} from './apply'; +import { + backoffDelayMs, escalationApplies, movesCursor, nextRung, ReplicaSyncError, rungValue, +} from './errors'; +import { + AnchorNotFoundError, captureSnapshotStates, getEmailChanges, getEmails, getMailboxChanges, + getMailboxes, maxObjectsInGet, queryAscending, +} from './jmap'; +import { + adjustForWindow, computeFloors, floorMovement, guardFloorAgainstClockJump, +} from './retention'; +import { CURSOR_TYPES, mintEnumerationCommitment, type CursorType } from './states'; +import { ReplicaStore, type RetentionPolicy } from './store'; +import type { BodyQueueEntry, CoverageState, EnvelopeRow, MailboxRow } from './types'; + +// ── budgets ────────────────────────────────────────────────────────────────── +// A cycle runs inside an HTTP request the renderer is waiting on, so the wall +// clock matters more than page counts. Budget exhaustion is a NORMAL outcome +// reported as `unfinishedWork`, never an error. +export const BUDGET = { + changesPagesPerCursor: 20, + coveragePages: 12, + coveragePageSize: 200, + bodyItems: 60, + bodyFetchChunk: 10, + envelopeFetchChunk: 50, + wallClockMs: 45_000, +} as const; + +export const MAX_BODY_ATTEMPTS = 5; +/** Reconciles per rolling 24 h before throttling. Never a hard stop. */ +export const MAX_RECONCILES_PER_DAY = 4; + +/** + * The reconcile stamp, derived from the DATA and not from the clock. + * + * Exported so it is testable on its own, because getting it wrong fails SILENTLY + * in both directions. With a frozen or coarse clock, a plain `now` leaves + * `cached_at < stamp` matching nothing and the sweep deletes nothing at all. And + * because the pin routinely EXCEEDS `now`, any enumeration path that stamps with + * `now` instead of the pin leaves its rows below the pin and gets them deleted by + * the very sweep that just re-verified them against the server. + */ +export function reconcileStamp(now: number, maxCachedAt: number): number { + return Math.max(now, maxCachedAt + 1); +} + +export interface CycleContext { + store: ReplicaStore; + session: JmapSessionInfo; + authHeader: string; + jmapAccountId: string; + policy: RetentionPolicy; + now: number; + deadline: number; +} + +export interface CycleReport { + ok: boolean; + /** True when budgets ran out or a queue still has wanted work - chain another cycle. */ + unfinishedWork: boolean; + bootstrapped: boolean; + reconciled: boolean; + mailboxesWritten: number; + envelopesWritten: number; + envelopesDeleted: number; + bodiesWritten: number; + bodiesEvicted: number; + coveragePhase: CoverageState['phase']; + resyncRequired: boolean; + warnings: string[]; + errorClass?: string; + error?: string; + retryAfterMs?: number; + durationMs: number; +} + +// ── row conversion ─────────────────────────────────────────────────────────── + +function toEnvelopeRow(jmapAccountId: string, email: Email): EnvelopeRow | null { + // `received_at` is NOT NULL and drives every window, so a record without one + // cannot be stored. In practice the server always sends it for a full envelope + // fetch; skipping is the safe response to a malformed one. + if (typeof email.receivedAt !== 'string' || email.receivedAt.length === 0) return null; + return { + jmapAccountId, + id: email.id, + threadId: email.threadId ?? null, + receivedAt: email.receivedAt, + size: typeof email.size === 'number' ? email.size : null, + subject: email.subject ?? null, + preview: email.preview ?? null, + fromJson: email.from ? JSON.stringify(email.from) : null, + toJson: email.to ? JSON.stringify(email.to) : null, + ccJson: email.cc ? JSON.stringify(email.cc) : null, + blobId: email.blobId ?? null, + hasAttachment: email.hasAttachment === true, + keywordsJson: JSON.stringify(email.keywords ?? {}), + // The BARE JMAP mailbox ids, never the display layer's `:` + // prefixed form. The store already keys every row by (jmapAccountId, id), so + // a prefixed id would double-encode the account and break every lookup. + mailboxIds: Object.entries(email.mailboxIds ?? {}) + .filter(([, v]) => v) + .map(([k]) => k), + }; +} + +function toMailboxRow(jmapAccountId: string, mailbox: Mailbox): MailboxRow { + return { + jmapAccountId, + id: mailbox.originalId ?? mailbox.id, + name: mailbox.name ?? '', + parentId: mailbox.parentId ?? null, + role: mailbox.role ?? null, + sortOrder: typeof mailbox.sortOrder === 'number' ? mailbox.sortOrder : null, + totalEmails: typeof mailbox.totalEmails === 'number' ? mailbox.totalEmails : null, + unreadEmails: typeof mailbox.unreadEmails === 'number' ? mailbox.unreadEmails : null, + totalThreads: typeof mailbox.totalThreads === 'number' ? mailbox.totalThreads : null, + unreadThreads: typeof mailbox.unreadThreads === 'number' ? mailbox.unreadThreads : null, + myRightsJson: mailbox.myRights ? JSON.stringify(mailbox.myRights) : null, + isSubscribed: mailbox.isSubscribed !== false, + }; +} + +/** The body tier as one opaque JSON blob. */ +export function serialiseBody(email: Email): string { + return JSON.stringify({ + sentAt: email.sentAt, + bcc: email.bcc, + replyTo: email.replyTo, + textBody: email.textBody, + htmlBody: email.htmlBody, + bodyValues: email.bodyValues, + attachments: email.attachments, + messageId: email.messageId, + inReplyTo: email.inReplyTo, + references: email.references, + headers: email.headers, + bodyStructure: email.bodyStructure, + }); +} + +// ── the cycle ──────────────────────────────────────────────────────────────── + +export async function runCycle(ctx: CycleContext): Promise { + const started = Date.now(); + const report: CycleReport = { + ok: true, + unfinishedWork: false, + bootstrapped: false, + reconciled: false, + mailboxesWritten: 0, + envelopesWritten: 0, + envelopesDeleted: 0, + bodiesWritten: 0, + bodiesEvicted: 0, + coveragePhase: 'never-run', + resyncRequired: false, + warnings: [], + durationMs: 0, + }; + + const { store, jmapAccountId, now } = ctx; + const flags = store.getFlags(now); + const rawFloors = computeFloors(ctx.policy, now); + + // Intent vs glitch. Without this discriminator the clock guard also fires on a + // legitimate user retention change, leaving a Settings edit unapplied until + // some unrelated trigger happened to move the floor again. + const policyChanged = + flags.lastEnvelopeDays !== undefined && flags.lastEnvelopeDays !== ctx.policy.envelopeDays; + const guarded = guardFloorAgainstClockJump(rawFloors.envelopeFrom, flags.lastWindowFloor, { + policyChanged, + }); + if (guarded.warning) { + report.warnings.push(guarded.warning); + logger.warn('offline-replica: retention floor suppressed', { warning: guarded.warning }); + } + const envelopeFrom = guarded.envelopeFrom; + // Clamp a second time after the guard: a suppressed envelope floor can end up + // NEWER than the computed body floor, and a body without an envelope is an orphan. + const bodyFrom = rawFloors.bodyFrom > envelopeFrom ? rawFloors.bodyFrom : envelopeFrom; + + try { + let coverage = store.getCoverage(jmapAccountId); + + // ── bootstrap / reconcile ──────────────────────────────────────────────── + if (!coverage || coverage.phase === 'never-run') { + await beginEnumeration(ctx, envelopeFrom, 'bootstrap'); + report.bootstrapped = true; + coverage = store.getCoverage(jmapAccountId); + } else if (flags.resyncRequired && coverage.phase !== 'reconciling') { + if (reconcileBudgetAllows(store, flags, now)) { + await beginEnumeration(ctx, envelopeFrom, 'reconcile'); + report.reconciled = true; + coverage = store.getCoverage(jmapAccountId); + } else { + // Throttle, never stop: a hard stop would trade a reconcile loop for a + // permanent wedge. + report.warnings.push('reconcile throttled: more than 4 rebuilds in the last 24h'); + } + } + + // ── A1 / A2: drain the two /changes cursors ────────────────────────────── + let anyCursorPending = false; + for (const type of CURSOR_TYPES) { + if (Date.now() > ctx.deadline) { report.unfinishedWork = true; break; } + const drained = await drainCursor(ctx, type, bodyFrom, report); + if (drained.pending) anyCursorPending = true; + } + if (anyCursorPending) report.unfinishedWork = true; + + // ── B: coverage enumeration ────────────────────────────────────────────── + coverage = store.getCoverage(jmapAccountId); + if (coverage && (coverage.phase === 'scanning' || coverage.phase === 'reconciling')) { + const scanned = await runCoverage(ctx, coverage, bodyFrom, guarded.evictionAllowed, report); + if (scanned.unfinished) report.unfinishedWork = true; + } + + // ── C1 / C2: bodies ────────────────────────────────────────────────────── + const drainedBodies = await drainBodyQueue(ctx, report); + const backfilled = await backfillBodies(ctx, bodyFrom, rawFloors.maxBodyBytes, drainedBodies); + if (backfilled > 0) report.unfinishedWork = true; + if (store.countWantedBodies(jmapAccountId, Date.now()) > 0) report.unfinishedWork = true; + + // ── retention ──────────────────────────────────────────────────────────── + applyRetention(ctx, { + envelopeFrom, + bodyFrom, + maxBodyBytes: rawFloors.maxBodyBytes, + evictionAllowed: guarded.evictionAllowed, + previousFloor: flags.lastWindowFloor, + report, + }); + + const finalCoverage = store.getCoverage(jmapAccountId); + const finalFlags = store.getFlags(now); + report.coveragePhase = finalCoverage?.phase ?? 'never-run'; + report.resyncRequired = finalFlags.resyncRequired; + + store.transaction(() => { + store.patchFlags(now, { + // The USED floor, never the rejected one. + lastWindowFloor: guarded.nextLastWindowFloor, + lastEnvelopeDays: ctx.policy.envelopeDays, + lastMaxBodyBytes: rawFloors.maxBodyBytes, + lastCycleAt: now, + lastCycleOk: true, + lastCycleError: undefined, + }); + }); + } catch (error) { + report.ok = false; + if (error instanceof ReplicaSyncError) { + report.errorClass = error.cls; + report.error = error.message; + report.retryAfterMs = error.retryAfterMs; + // "Offline is not an error" - a transport failure leaves every cursor + // exactly where it was and is simply retried later. + report.unfinishedWork = error.cls !== 'Fatal' && error.cls !== 'Auth'; + } else { + report.error = error instanceof Error ? error.message : String(error); + report.unfinishedWork = false; + } + try { + store.transaction(() => { + store.patchFlags(now, { lastCycleAt: now, lastCycleOk: false, lastCycleError: report.error }); + }); + } catch { /* the store may be the thing that failed */ } + } + + report.durationMs = Date.now() - started; + return report; +} + +// ── bootstrap / reconcile ──────────────────────────────────────────────────── + +function reconcileBudgetAllows( + store: ReplicaStore, + flags: { reconcilesInWindow: number; reconcileWindowStartedAt: number }, + now: number, +): boolean { + const dayMs = 24 * 60 * 60 * 1000; + if (now - flags.reconcileWindowStartedAt > dayMs) { + store.transaction(() => { + store.patchFlags(now, { reconcilesInWindow: 0, reconcileWindowStartedAt: now }); + }); + return true; + } + return flags.reconcilesInWindow < MAX_RECONCILES_PER_DAY; +} + +/** + * THE MANDATORY ORDER. Step 1 must precede step 3. + * + * 1. capture both cursors, in one request, BEFORE touching any data, and seed + * them inside one EnumerationCommitment that writes the coverage row in the + * SAME transaction - so a seed is never durable without the durable promise to + * enumerate that justifies it. + * 2. full `Mailbox/get` - cheap, always complete, no paging. + * 3. the seeded cursors are LIVE FROM HERE. Each cycle runs the delta jobs and + * only then the scan, so a wide-window rebuild does not stall incoming mail. + * 4. when the scan reaches the target: `coveredFrom = sweepFloor`, phase complete. + */ +async function beginEnumeration( + ctx: CycleContext, + envelopeFrom: string, + kind: 'bootstrap' | 'reconcile', +): Promise { + const { store, jmapAccountId, now } = ctx; + + // Step 0 for a reconcile: PIN THE FLOOR. Every later step reads `sweepFloor`, + // never a live `targetFrom`. Widening retention while a reconcile runs would + // otherwise make the sweep delete against the new wide window when the + // enumeration only covered the old narrow one - permanently, since `coveredFrom` + // then claims the wider range and `/changes` cannot re-deliver old mail. + const existing = store.getCoverage(jmapAccountId); + const sweepFloor = kind === 'reconcile' ? (existing?.deferredTargetFrom ?? envelopeFrom) : envelopeFrom; + + const snapshots = await captureSnapshotStates(ctx.session, ctx.authHeader, jmapAccountId); + + // Derive the reconcile stamp from the DATA, not the clock: with a frozen or + // coarse clock `cached_at < stamp` matches nothing and the sweep deletes nothing. + const stampedAt = kind === 'reconcile' + ? reconcileStamp(now, store.maxEnvelopeCachedAt(jmapAccountId)) + : undefined; + + store.transaction(() => { + for (const type of CURSOR_TYPES) { + store.seedCursor( + { jmapAccountId, type }, + mintEnumerationCommitment({ + jmapAccountId, + snapshot: type === 'Mailbox' ? snapshots.mailbox : snapshots.email, + targetFrom: envelopeFrom, + sweepFloor, + kind, + }), + now, + ); + } + if (stampedAt !== undefined) store.patchCoverage(jmapAccountId, { reconcileStampedAt: stampedAt }); + if (kind === 'reconcile') { + const flags = store.getFlags(now); + store.patchFlags(now, { reconcilesInWindow: flags.reconcilesInWindow + 1 }); + } + }); + + // Step 2: every mailbox, in full. + const mailboxes = await getMailboxes(ctx.session, ctx.authHeader, jmapAccountId, null); + const rows = mailboxes.map((m) => toMailboxRow(jmapAccountId, m)); + store.transaction(() => { store.upsertMailboxes(rows); }); +} + +// ── A1 / A2: the delta drain ───────────────────────────────────────────────── + +async function drainCursor( + ctx: CycleContext, + type: CursorType, + bodyFrom: string, + report: CycleReport, +): Promise<{ pending: boolean }> { + const { store, jmapAccountId } = ctx; + const key = { jmapAccountId, type }; + let cursor = store.getCursor(key); + if (!cursor) return { pending: false }; + if (cursor.invalidatedAt) { + // Serving `/changes` from an invalidated cursor is forbidden. The reconcile + // is what clears it. + return { pending: false }; + } + + const cap = maxObjectsInGet(ctx.session); + let pages = 0; + + while (pages < BUDGET.changesPagesPerCursor) { + if (Date.now() > ctx.deadline) { + store.transaction(() => { store.patchCursor(key, { drainPending: true }); }); + return { pending: true }; + } + cursor = store.getCursor(key); + if (!cursor) return { pending: false }; + + let page; + try { + const maxChanges = rungValue(cursor.maxChangesRung, cap); + page = type === 'Email' + ? await getEmailChanges(ctx.session, ctx.authHeader, jmapAccountId, cursor.state, maxChanges) + : await getMailboxChanges(ctx.session, ctx.authHeader, jmapAccountId, cursor.state, maxChanges); + } catch (error) { + handleDrainError(ctx, key, cursor.state, error, report); + // Any cursor failing means the cycle is unfinished for escalation + // purposes, but never that the cursor moved. + return { pending: true }; + } + pages++; + + if (page.oldState !== cursor.state) { + // Re-issue once before escalating: a transient anomaly is far more common + // than a genuine invalidation, and a full rebuild is expensive. + let reissued; + try { + const maxChanges = rungValue(cursor.maxChangesRung, cap); + reissued = type === 'Email' + ? await getEmailChanges(ctx.session, ctx.authHeader, jmapAccountId, cursor.state, maxChanges) + : await getMailboxChanges(ctx.session, ctx.authHeader, jmapAccountId, cursor.state, maxChanges); + } catch (error) { + handleDrainError(ctx, key, cursor.state, error, report); + return { pending: true }; + } + if (reissued.oldState !== cursor.state) { + invalidate(ctx, key, 'oldStateMismatch', report); + return { pending: false }; + } + // Use the RE-ISSUED page. Advancing to the original response's newState + // would skip whatever the re-issue reported - the same silent-gap shape as + // the cursor-provenance bug, reintroduced one level down. + page = reissued; + report.warnings.push(`${type}/changes reported a transient oldState mismatch`); + } + + if (pageIsEmpty(page)) { + // An empty page still advances. Skipping it re-requests forever. + store.transaction(() => { + store.advanceCursor(key, page.newState); + store.patchCursor(key, { consecutiveFailures: 0, lastFailedState: undefined, maxChangesRung: 0 }); + }); + if (!page.hasMoreChanges) { + store.transaction(() => { store.patchCursor(key, { drainPending: false }); }); + return { pending: false }; + } + continue; + } + + if (type === 'Mailbox') { + await applyMailboxPage(ctx, page, report); + } else { + await applyEmailPage(ctx, page, bodyFrom, report); + } + + store.transaction(() => { + store.advanceCursor(key, page.newState); + store.patchCursor(key, { + consecutiveFailures: 0, + lastFailedState: undefined, + maxChangesRung: 0, + drainPending: page.hasMoreChanges, + }); + }); + + if (!page.hasMoreChanges) return { pending: false }; + } + + // Budget exhaustion is normal, not an error. It is also the answer to a server + // whose `hasMoreChanges` never goes false. + store.transaction(() => { store.patchCursor(key, { drainPending: true }); }); + return { pending: true }; +} + +function handleDrainError( + ctx: CycleContext, + key: { jmapAccountId: string; type: CursorType }, + failedState: string, + error: unknown, + report: CycleReport, +): void { + const { store } = ctx; + const cls = error instanceof ReplicaSyncError ? error.cls : 'ServerTransient'; + const message = error instanceof Error ? error.message : String(error); + report.warnings.push(`${key.type}/changes ${cls}: ${message}`); + if (error instanceof ReplicaSyncError && error.retryAfterMs) { + report.retryAfterMs = Math.max(report.retryAfterMs ?? 0, error.retryAfterMs); + } + + if (movesCursor(cls)) { + invalidate(ctx, key, 'cannotCalculateChanges', report); + return; + } + + const cursor = store.getCursor(key); + if (!cursor) return; + // The ladder only counts failures at the SAME sinceState: a failure at a new + // position means progress was made, so the ladder restarts. + const sameSpot = cursor.lastFailedState === failedState; + const failures = sameSpot ? cursor.consecutiveFailures + 1 : 1; + store.transaction(() => { + store.patchCursor(key, { + consecutiveFailures: failures, + lastFailedState: failedState, + maxChangesRung: escalationApplies(cls) && failures >= 2 + ? nextRung(cursor.maxChangesRung) + : cursor.maxChangesRung, + }); + }); +} + +/** + * RFC 8620 s5.2 says the client MUST invalidate its cache. The LITERAL reading - + * delete everything, now - would empty a user's offline mail exactly when they may + * be offline and depending on it. So: mark the cursor unusable, set the STICKY + * resync flag, and leave every record READABLE. The accepted cost is that between + * detection and the sweep, a server-deleted message can still show locally. + * + * An invalidation of EITHER cursor reconciles the account as a whole - splitting + * it is not worth the reasoning burden when `Mailbox/get` is one cheap call. + */ +function invalidate( + ctx: CycleContext, + key: { jmapAccountId: string; type: CursorType }, + reason: 'cannotCalculateChanges' | 'oldStateMismatch', + report: CycleReport, +): void { + const { store, now } = ctx; + store.transaction(() => { + store.patchCursor(key, { invalidatedAt: now, invalidatedReason: reason }); + store.patchFlags(now, { resyncRequired: true }); + }); + report.resyncRequired = true; + report.warnings.push(`${key.type} cursor invalidated (${reason}); a rebuild is queued`); + logger.warn('offline-replica: cursor invalidated', { type: key.type, reason }); +} + +async function applyMailboxPage( + ctx: CycleContext, + page: Parameters[0], + report: CycleReport, +): Promise { + const { store, jmapAccountId } = ctx; + const plan = planMailboxFetches(page); + + if (plan.fullIds.length > 0) { + const mailboxes = await getMailboxes(ctx.session, ctx.authHeader, jmapAccountId, plan.fullIds); + const rows = mailboxes.map((m) => toMailboxRow(jmapAccountId, m)); + store.transaction(() => { report.mailboxesWritten += store.upsertMailboxes(rows); }); + } + + if (plan.countOnlyIds.length > 0) { + const props = ['totalEmails', 'unreadEmails', 'totalThreads', 'unreadThreads']; + const mailboxes = await getMailboxes( + ctx.session, ctx.authHeader, jmapAccountId, plan.countOnlyIds, props, + ); + store.transaction(() => { + for (const m of mailboxes) { + store.patchMailboxCounts(jmapAccountId, m.originalId ?? m.id, { + totalEmails: typeof m.totalEmails === 'number' ? m.totalEmails : undefined, + unreadEmails: typeof m.unreadEmails === 'number' ? m.unreadEmails : undefined, + totalThreads: typeof m.totalThreads === 'number' ? m.totalThreads : undefined, + unreadThreads: typeof m.unreadThreads === 'number' ? m.unreadThreads : undefined, + }); + } + }); + } + + if (plan.destroyIds.length > 0) { + // The mailbox row ONLY. Never email records. + store.transaction(() => { store.deleteMailboxes(jmapAccountId, plan.destroyIds); }); + } +} + +async function applyEmailPage( + ctx: CycleContext, + page: Parameters[0], + bodyFrom: string, + report: CycleReport, +): Promise { + const { store, jmapAccountId, now } = ctx; + // Presence is tested in BULK, before either fetch is issued - an `updated` id we + // do not hold is filtered out rather than fetched and then discarded. + const present = store.whichEnvelopesExist(jmapAccountId, page.updated); + const plan = planEmailFetches(page, present); + + // CREATES: full envelope tier, and a body enqueue when inside the body window. + // Bodies are NEVER fetched inline - they are 10-500 KB against an envelope's + // ~1 KB, and the queue is what keeps a page a small, quickly-committable unit. + for (let i = 0; i < plan.createIds.length; i += BUDGET.envelopeFetchChunk) { + const chunk = plan.createIds.slice(i, i + BUDGET.envelopeFetchChunk); + const { list } = await getEmails(ctx.session, ctx.authHeader, jmapAccountId, chunk, 'envelope'); + const rows = list.map((e) => toEnvelopeRow(jmapAccountId, e)).filter((r): r is EnvelopeRow => r !== null); + const queue: BodyQueueEntry[] = rows + .filter((r) => r.receivedAt >= bodyFrom) + .map((r) => ({ emailId: r.id, jmapAccountId, receivedAt: r.receivedAt, attempts: 0 })); + store.transaction(() => { + report.envelopesWritten += store.upsertEnvelopes(rows, now); + store.enqueueBodies(queue); + }); + } + + // UPDATES: three properties, never a body. + for (let i = 0; i < plan.updateIds.length; i += BUDGET.envelopeFetchChunk) { + const chunk = plan.updateIds.slice(i, i + BUDGET.envelopeFetchChunk); + const { list } = await getEmails(ctx.session, ctx.authHeader, jmapAccountId, chunk, 'mutable'); + store.transaction(() => { + for (const e of list) { + store.patchEnvelopeMutable(jmapAccountId, e.id, { + keywordsJson: JSON.stringify(e.keywords ?? {}), + mailboxIds: Object.entries(e.mailboxIds ?? {}).filter(([, v]) => v).map(([k]) => k), + }); + } + }); + } + + // DESTROYS LAST. Ids are never reused, so a destroy always refers to the same + // record as any create/update of that id in the same page, and destroy-last + // converges. The reverse order would resurrect a dead id, spend a fetch and get + // `notFound`. A destroy for an id we never held is a harmless no-op. + if (plan.destroyIds.length > 0) { + store.transaction(() => { + report.envelopesDeleted += store.deleteEmails(jmapAccountId, plan.destroyIds); + }); + } +} + +// ── B: coverage enumeration ────────────────────────────────────────────────── + +async function runCoverage( + ctx: CycleContext, + coverage: CoverageState, + bodyFrom: string, + evictionAllowed: boolean, + report: CycleReport, +): Promise<{ unfinished: boolean }> { + const { store, jmapAccountId, now } = ctx; + const isReconcile = coverage.phase === 'reconciling'; + const floor = isReconcile ? (coverage.sweepFloor ?? coverage.targetFrom) : coverage.targetFrom; + // A reconcile stamps with its PINNED value, not `now`. The pin is + // max(now, maxCachedAt + 1) so it routinely EXCEEDS now - stamping with `now` + // would leave rows below the pin and get them deleted by the very sweep that + // just re-verified them against the server. + const stamp = isReconcile ? coverage.reconcileStampedAt ?? now : now; + + let scanCursor = coverage.scanCursor ?? floor; + let lastPageIds: string[] = []; + let pages = 0; + + while (pages < BUDGET.coveragePages) { + if (Date.now() > ctx.deadline) return { unfinished: true }; + pages++; + + let ids: string[]; + try { + ({ ids } = await queryAscending( + ctx.session, ctx.authHeader, jmapAccountId, scanCursor, BUDGET.coveragePageSize, + )); + } catch (error) { + if (error instanceof AnchorNotFoundError) { + ids = []; + } else { + store.transaction(() => { + store.patchCoverage(jmapAccountId, { + consecutiveFailures: coverage.consecutiveFailures + 1, + }); + }); + throw error; + } + } + + if (ids.length === 0) { + // The scan has reached the present. + finishEnumeration(ctx, floor, stamp, isReconcile, evictionAllowed, report); + return { unfinished: !evictionAllowed && isReconcile }; + } + + const { list } = await getEmails(ctx.session, ctx.authHeader, jmapAccountId, ids, 'envelope'); + const rows = list.map((e) => toEnvelopeRow(jmapAccountId, e)).filter((r): r is EnvelopeRow => r !== null); + const maxReceivedAt = rows.reduce( + (max, r) => (max === null || r.receivedAt > max ? r.receivedAt : max), + null, + ); + const queue: BodyQueueEntry[] = rows + .filter((r) => r.receivedAt >= bodyFrom) + .map((r) => ({ emailId: r.id, jmapAccountId, receivedAt: r.receivedAt, attempts: 0 })); + + let nextScanCursor = scanCursor; + let gapMarker: NonNullable[number] | null = null; + + if (madeForwardProgress(maxReceivedAt, scanCursor)) { + nextScanCursor = maxReceivedAt as string; + } else if (ids.length >= BUDGET.coveragePageSize) { + // A FULL page whose every row shares one millisecond. Try the anchor rung + // first: resume from the id after the last one we saw. + const anchorId = lastPageIds.length > 0 ? lastPageIds[lastPageIds.length - 1] : ids[ids.length - 1]; + let recovered = false; + try { + const anchored = await queryAscending( + ctx.session, ctx.authHeader, jmapAccountId, scanCursor, BUDGET.coveragePageSize, + { anchor: anchorId, anchorOffset: 1 }, + ); + if (anchored.ids.length > 0) { + const fetched = await getEmails( + ctx.session, ctx.authHeader, jmapAccountId, anchored.ids, 'envelope', + ); + const anchoredRows = fetched.list + .map((e) => toEnvelopeRow(jmapAccountId, e)) + .filter((r): r is EnvelopeRow => r !== null); + const anchoredMax = anchoredRows.reduce( + (max, r) => (max === null || r.receivedAt > max ? r.receivedAt : max), + null, + ); + store.transaction(() => { + // The PINNED stamp here too, for the same reason. + report.envelopesWritten += store.upsertEnvelopes(anchoredRows, stamp); + store.enqueueBodies( + anchoredRows + .filter((r) => r.receivedAt >= bodyFrom) + .map((r) => ({ emailId: r.id, jmapAccountId, receivedAt: r.receivedAt, attempts: 0 })), + ); + }); + if (anchoredMax !== null && anchoredMax > scanCursor) { + nextScanCursor = anchoredMax; + recovered = true; + } + } + } catch (error) { + if (!(error instanceof AnchorNotFoundError)) throw error; + } + if (!recovered) { + // Last resort: advance one millisecond, WARN, and leave a durable trace. + // This rung CAN skip messages sharing the boundary millisecond, so it is + // never normal-path behaviour and always leaves a record so a support + // question has an answer. A 200-message single-millisecond cluster is a + // corrupt server, not a case to design for. + const to = advanceOneMs(scanCursor); + gapMarker = { from: scanCursor, to, reason: 'tie-cluster-skip', at: now }; + nextScanCursor = to; + report.warnings.push( + `coverage skipped a tie cluster at ${scanCursor}; some messages sharing that ` + + `millisecond may be missing from the offline store`, + ); + logger.warn('offline-replica: tie-cluster skip', { at: scanCursor }); + } + } else { + // A partial page with no forward progress means we are at the tail. + // + // COMMIT THE ROWS BEFORE FINISHING. Finishing runs the reconcile sweep, and + // the sweep deletes anything still stamped below the pin - so finishing + // first would delete this page's own records (taking their bodies and queue + // rows with them) and then re-insert them bodyless, costing a re-download of + // every body at the tail on every reconcile. + store.transaction(() => { + report.envelopesWritten += store.upsertEnvelopes(rows, stamp); + store.enqueueBodies(queue); + }); + finishEnumeration(ctx, floor, stamp, isReconcile, evictionAllowed, report); + return { unfinished: false }; + } + + const advanceTo = nextScanCursor; + const marker = gapMarker; + store.transaction(() => { + report.envelopesWritten += store.upsertEnvelopes(rows, stamp); + store.enqueueBodies(queue); + // CURSOR LAST, inside the same transaction as the records it accounts for. + store.patchCoverage(jmapAccountId, { + scanCursor: advanceTo, + seen: coverage.seen + rows.length, + consecutiveFailures: 0, + ...(marker + ? { gapMarkers: [...(coverage.gapMarkers ?? []), marker].slice(-32) } + : {}), + }); + }); + scanCursor = advanceTo; + lastPageIds = ids; + } + + return { unfinished: true }; +} + +/** + * The only place the reconcile pins are released, and it clears them all at once. + */ +function finishEnumeration( + ctx: CycleContext, + floor: string, + stamp: number, + isReconcile: boolean, + evictionAllowed: boolean, + report: CycleReport, +): void { + const { store, jmapAccountId, now } = ctx; + const coverage = store.getCoverage(jmapAccountId); + if (!coverage) return; + + if (isReconcile && !evictionAllowed) { + // The sweep is a delete against the retention floor, so it is subject to the + // same rule as eviction: a suspect clock reading may not drive deletion. + // Leave the reconcile open; it completes on a cycle whose floor is trustworthy. + report.warnings.push( + 'deferring the reconcile sweep: the retention floor came from a suppressed clock anomaly', + ); + return; + } + + store.transaction(() => { + if (isReconcile) { + report.envelopesDeleted += store.sweep(jmapAccountId, floor, stamp); + // A give-up recorded during whatever went wrong must not outlive it, or a + // transient outage would permanently deny a body with no path back. + store.clearBodyGiveUps(jmapAccountId); + store.patchFlags(now, { resyncRequired: false }); + for (const type of CURSOR_TYPES) { + store.patchCursor({ jmapAccountId, type }, { + invalidatedAt: undefined, + invalidatedReason: undefined, + consecutiveFailures: 0, + lastFailedState: undefined, + maxChangesRung: 0, + }); + } + } + const deferred = coverage.deferredTargetFrom; + store.patchCoverage(jmapAccountId, { + coveredFrom: floor, + scanCursor: null, + sweepFloor: undefined, + reconcileStampedAt: undefined, + deferredTargetFrom: undefined, + ...(deferred ? { targetFrom: deferred, phase: 'scanning' as const } : { phase: 'complete' as const }), + }); + }); +} + +// ── C1: drain the durable body queue ───────────────────────────────────────── + +async function drainBodyQueue(ctx: CycleContext, report: CycleReport): Promise { + const { store, jmapAccountId } = ctx; + let fetched = 0; + const wanted = store.takeBodyQueue(jmapAccountId, BUDGET.bodyItems, Date.now()); + if (wanted.length === 0) return 0; + + for (let i = 0; i < wanted.length; i += BUDGET.bodyFetchChunk) { + if (Date.now() > ctx.deadline) { report.unfinishedWork = true; break; } + const chunk = wanted.slice(i, i + BUDGET.bodyFetchChunk); + let result; + try { + result = await getEmails( + ctx.session, ctx.authHeader, jmapAccountId, chunk.map((e) => e.emailId), 'body', + ); + } catch (error) { + // Body failures NEVER touch a cursor: the body jobs are separate state, so + // the delta path keeps its position and the queue simply retries later. + const message = error instanceof Error ? error.message : String(error); + store.transaction(() => { + for (const e of chunk) { + store.bumpBodyAttempt(jmapAccountId, e.emailId, Date.now() + backoffDelayMs(e.attempts), message); + } + }); + report.warnings.push(`body fetch failed: ${message}`); + report.unfinishedWork = true; + break; + } + + const byId = new Map(result.list.map((e) => [e.id, e])); + const notFound = new Set(result.notFound); + + store.transaction(() => { + for (const entry of chunk) { + const email = byId.get(entry.emailId); + if (email) { + // Conditional on the envelope still existing, so a body fetched just + // before its envelope was destroyed cannot land as an orphan. + const wrote = store.putBodyIfEnvelopeExists(jmapAccountId, entry.emailId, serialiseBody(email)); + store.dequeueBodies(jmapAccountId, [entry.emailId]); + if (wrote) { fetched++; report.bodiesWritten++; } + continue; + } + if (notFound.has(entry.emailId)) { + // The message is gone, so the entry can NEVER succeed and must not burn + // five attempts. A durable give-up, not a row deletion - deleting it + // would let the backfill pass re-insert it every cycle forever. + store.markBodyGaveUp(jmapAccountId, [ + { emailId: entry.emailId, receivedAt: entry.receivedAt, reason: 'notFound' }, + ]); + continue; + } + // Present in neither list nor notFound: a transient miss. + const attempts = entry.attempts + 1; + if (attempts >= MAX_BODY_ATTEMPTS) { + store.markBodyGaveUp(jmapAccountId, [ + { + emailId: entry.emailId, + receivedAt: entry.receivedAt, + reason: 'attempts', + lastError: `gave up after ${attempts} attempts`, + }, + ]); + } else { + store.bumpBodyAttempt( + jmapAccountId, entry.emailId, Date.now() + backoffDelayMs(attempts), + 'the server returned neither the record nor notFound', + ); + } + } + }); + } + return fetched; +} + +// ── C2: body backfill ─────────────────────────────────────────────────────── + +/** + * Notices envelopes that never had a body enqueued - a widened body window, or a + * queue row lost to a purge. Runs EVEN WHEN C1 found nothing, because that is its + * whole job. + * + * It is cap-aware and excludes durable give-ups. Both matter: the headroom check + * avoids paying for a download with nowhere to go, and the give-up exclusion is + * what makes termination provable. Without it, the size cap sheds the oldest + * bodies, those envelopes are still inside the body WINDOW, so this pass + * re-enqueues them, they download again, and the cap sheds them again - unbounded + * data use with no termination, because there is always an envelope without a body. + * + * A heuristic frontier instead of a durable mark does NOT work, and the reason is + * worth recording: the evictor sheds down TO the cap, which leaves headroom, so + * this pass refills and the two trade the same bytes back and forth. Only the + * durable mark makes the store monotone. + */ +async function backfillBodies( + ctx: CycleContext, + bodyFrom: string, + maxBodyBytes: number, + alreadyFetched: number, +): Promise { + const { store, jmapAccountId } = ctx; + const budget = BUDGET.bodyItems - alreadyFetched; + if (budget <= 0) return 0; + + const used = store.bodyBytesTotal(jmapAccountId); + let headroom = maxBodyBytes - used; + if (headroom <= 0) return 0; + + const candidates = store.envelopesWithoutBody(jmapAccountId, bodyFrom, budget * 2); + if (candidates.length === 0) return 0; + const gaveUp = new Set(store.listBodyGiveUps(jmapAccountId, 5_000)); + + const entries: BodyQueueEntry[] = []; + for (const c of candidates) { + if (entries.length >= budget) break; + if (gaveUp.has(c.id)) continue; + // Always allow at least one, or a single oversized message would stall the + // queue forever. + if (entries.length > 0 && c.size > headroom) break; + headroom -= c.size; + entries.push({ emailId: c.id, jmapAccountId, receivedAt: c.receivedAt, attempts: 0 }); + if (headroom <= 0) break; + } + if (entries.length === 0) return 0; + + // The INSERTED count, not the attempted count. Reporting the latter made the + // mobile engine treat every cycle as having unfinished work for as long as any + // envelope lacked a body, chaining a new cycle every few seconds indefinitely. + return store.transaction(() => store.enqueueBodies(entries)); +} + +// ── retention ──────────────────────────────────────────────────────────────── + +function applyRetention( + ctx: CycleContext, + args: { + envelopeFrom: string; + bodyFrom: string; + maxBodyBytes: number; + evictionAllowed: boolean; + previousFloor: string | undefined; + report: CycleReport; + }, +): void { + const { store, jmapAccountId, now } = ctx; + const { envelopeFrom, bodyFrom, maxBodyBytes, evictionAllowed, previousFloor, report } = args; + const coverage = store.getCoverage(jmapAccountId); + + // A retention change arriving DURING a reconcile is deferred to after the + // sweep. Applying a widen mid-reconcile would make the sweep delete everything + // between the old and new floors permanently; a narrow is deferred too, because + // evicting below a floor the sweep is about to use races it for no benefit. + if (coverage?.phase === 'reconciling') { + if (coverage.targetFrom !== envelopeFrom) { + store.transaction(() => { + store.patchCoverage(jmapAccountId, { deferredTargetFrom: envelopeFrom }); + }); + } + return; + } + + const movement = floorMovement(previousFloor, envelopeFrom); + const adjustment = adjustForWindow(movement, envelopeFrom); + + if (adjustment.evictBelow) { + if (!evictionAllowed) { + report.warnings.push( + `skipping retention eviction below ${adjustment.evictBelow}: the floor came from a ` + + `suppressed clock anomaly`, + ); + } else { + store.transaction(() => { + report.envelopesDeleted += store.evictEnvelopesBelow(jmapAccountId, adjustment.evictBelow as string); + // `coveredFrom` follows the floor, but ONLY when it was already set: + // otherwise a narrow claims a range that was never enumerated, and delta + // sync cannot re-deliver pre-existing mail to repair that. + if (coverage?.coveredFrom !== null && coverage?.coveredFrom !== undefined) { + store.patchCoverage(jmapAccountId, { coveredFrom: adjustment.evictBelow as string }); + } + store.patchCoverage(jmapAccountId, { targetFrom: envelopeFrom }); + }); + } + } else if (adjustment.rescanFrom) { + // A WIDEN is not a resync: the target moves back, coverage re-enters scanning, + // and the cursors are untouched. + store.transaction(() => { + store.patchCoverage(jmapAccountId, { + targetFrom: adjustment.rescanFrom as string, + scanCursor: adjustment.rescanFrom as string, + phase: 'scanning', + }); + }); + report.unfinishedWork = true; + } + + // Body narrow: delete and merely DEQUEUE, never mark - the backfill's + // `receivedAfter` already excludes out-of-window bodies, and a later widen must + // be free to re-fetch them. + const stale = store.bodiesBelow(jmapAccountId, bodyFrom, 2_000); + if (stale.length > 0 && evictionAllowed) { + store.transaction(() => { + report.bodiesEvicted += store.deleteBodies(jmapAccountId, stale); + store.dequeueBodies(jmapAccountId, stale); + }); + } + + // Orphan bodies: invisible to cap eviction, which only walks the body table. + const orphans = store.orphanBodies(jmapAccountId, 500); + if (orphans.length > 0) { + store.transaction(() => { + report.bodiesEvicted += store.deleteBodies(jmapAccountId, orphans); + store.dequeueBodies(jmapAccountId, orphans); + }); + } + + // A cap RAISE revives bodies shed for space: a durable refusal recorded under + // one policy must not outlive that policy. + const flags = store.getFlags(now); + if (flags.lastMaxBodyBytes !== undefined && maxBodyBytes > flags.lastMaxBodyBytes) { + store.transaction(() => { store.clearBodyGiveUps(jmapAccountId, 'shed-by-cap'); }); + } + + // The MB cap: oldest bodies first, envelopes always survive, so the message + // stays listed and openable when back online. + let used = store.bodyBytesTotal(jmapAccountId); + if (used > maxBodyBytes) { + const oldest = store.oldestBodies(jmapAccountId, 5_000); + const shed: Array<{ emailId: string; receivedAt: string; reason: 'shed-by-cap' }> = []; + for (const b of oldest) { + if (used <= maxBodyBytes) break; + used -= b.bytes; + shed.push({ emailId: b.emailId, receivedAt: '', reason: 'shed-by-cap' }); + } + if (shed.length > 0) { + // MARKED, not merely dequeued. See backfillBodies' comment for the loop + // this closes. + const withDates = shed.map((s) => { + const row = store.getEnvelopeRaw(jmapAccountId, s.emailId); + return { ...s, receivedAt: typeof row?.received_at === 'string' ? row.received_at : new Date(now).toISOString() }; + }); + store.transaction(() => { + report.bodiesEvicted += store.deleteBodies(jmapAccountId, shed.map((s) => s.emailId)); + store.markBodyGaveUp(jmapAccountId, withDates); + }); + } + } +} diff --git a/lib/offline-replica/types.ts b/lib/offline-replica/types.ts new file mode 100644 index 00000000..884ffc4b --- /dev/null +++ b/lib/offline-replica/types.ts @@ -0,0 +1,169 @@ +// Persisted shapes for the offline replica. +// +// Note that `SyncCursor.state` is declared as a plain `string` here while +// `./states.ts` goes to some trouble to brand it. That is deliberate: a token +// that has round-tripped through JSON has no provenance left to certify. The +// brands guard the WRITE PATHS (`advanceCursor` / `seedCursor`), which is where +// provenance is actually decided; the row is just a row. + +import type { CursorType } from './states'; + +export interface SyncCursor { + type: CursorType; + jmapAccountId: string; + /** A ChangesState from this (type, jmapAccountId), or a seeded SnapshotState. */ + state: string; + /** True when the last page reported `hasMoreChanges` - a drain is unfinished. */ + drainPending: boolean; + /** Set when the server invalidated us. Cleared only by a COMPLETED reconcile. */ + invalidatedAt?: number; + invalidatedReason?: 'cannotCalculateChanges' | 'oldStateMismatch' | 'corruptState' | 'manual'; + /** + * Anti-wedge counters, PER CURSOR rather than per account. With the counters + * shared, a healthy Mailbox cursor resetting them every cycle meant a failing + * Email cursor never escalated and never advanced again - silently, forever. + */ + consecutiveFailures: number; + /** The `sinceState` that failed; escalation only counts failures at the same position. */ + lastFailedState?: string; + /** Current rung of the `maxChanges` ladder. */ + maxChangesRung: 0 | 1 | 2 | 3; + updatedAt: number; +} + +export interface CoverageState { + jmapAccountId: string; + /** ISO. Oldest `receivedAt` for which the ENVELOPE tier is known-complete. */ + coveredFrom: string | null; + /** ISO. Ascending scan resume point; null when not scanning. */ + scanCursor: string | null; + /** The retention floor this scan is working toward. */ + targetFrom: string; + /** The floor PINNED at reconcile start. The sweep deletes only against this. */ + sweepFloor?: string; + /** Set when a retention change arrived mid-reconcile; applied after the sweep. */ + deferredTargetFrom?: string; + /** + * The `cached_at` stamp a reconcile's enumeration writes onto every envelope it + * re-sees, pinned when the reconcile starts. + * + * This is a multi-cycle, crash-resumable "seen set" implemented as ONE + * INTEGER, with no seen-ids table: the enumeration re-upserts each surviving + * envelope, refreshing its `cached_at`, so the sweep is + * `received_at >= sweepFloor AND cached_at < reconcileStampedAt`. A record the + * enumeration never reached keeps its older stamp and is swept; a record the + * LIVE delta path creates mid-reconcile gets `Date.now() >= stamp` and + * survives, which is exactly right. + * + * The stamp must be derived from the DATA, not the clock: + * `max(now, maxCachedAt + 1)`. With a frozen or coarse clock, `cached_at < + * stamp` matches nothing and the sweep deletes nothing. + */ + reconcileStampedAt?: number; + /** Durable trace of any tie-cluster skip taken by the last-resort paging rung. */ + gapMarkers?: Array<{ from: string; to: string; reason: 'tie-cluster-skip'; at: number }>; + phase: 'never-run' | 'scanning' | 'reconciling' | 'complete'; + /** Progress, for the UI only. Never load-bearing. */ + seen: number; + consecutiveFailures: number; + updatedAt: number; +} + +export interface BodyQueueEntry { + emailId: string; + jmapAccountId: string; + /** Drives priority: newest first. */ + receivedAt: string; + /** NEVER reset by a re-enqueue. */ + attempts: number; + lastError?: string; + nextAttemptAt?: number; + /** + * Durable terminal state. A gave-up row is KEPT rather than deleted, precisely + * so the backfill job cannot resurrect it - its driver is "envelope without a + * body", which by itself cannot distinguish "not fetched yet" from + * "deliberately not kept". Cleared wholesale by a completed reconcile, so a + * transient outage self-heals. + */ + gaveUp?: boolean; + gaveUpReason?: 'attempts' | 'notFound' | 'shed-by-cap'; +} + +export type BodyGiveUpReason = NonNullable; + +/** Per-account flags. A VIEW over field-level patches, never written whole. */ +export interface ReplicaFlags { + /** Sticky until a reconcile completes. Survives restarts. */ + resyncRequired: boolean; + /** Rolling count + window start for the reconcile ceiling. */ + reconcilesInWindow: number; + reconcileWindowStartedAt: number; + /** + * Last observed retention floor, for the clock-jump guard. + * + * This must hold the floor that was actually USED, never the suspicious one + * that was rejected - see `retention.ts` for the wipe that the other choice + * caused. + */ + lastWindowFloor?: string; + /** + * The `envelopeDays` that produced `lastWindowFloor`. + * + * The computed floor moves for TWO independent reasons - the clock changing + * and the SETTING changing - and guarding a setting change is wrong: it is + * explicit user intent, not a glitch. Recording the policy alongside the floor + * is what tells them apart. + */ + lastEnvelopeDays?: number; + /** + * The body-tier byte cap in force last cycle. A RAISE must revive bodies + * previously shed for space, which is otherwise a durable refusal. + */ + lastMaxBodyBytes?: number; + lastCycleAt?: number; + lastCycleOk?: boolean; + lastCycleError?: string; +} + +export function defaultFlags(now: number): ReplicaFlags { + return { + resyncRequired: false, + reconcilesInWindow: 0, + reconcileWindowStartedAt: now, + }; +} + +export type FlagsPatch = Partial; + +/** The envelope tier, as stored. */ +export interface EnvelopeRow { + jmapAccountId: string; + id: string; + threadId: string | null; + receivedAt: string; + size: number | null; + subject: string | null; + preview: string | null; + fromJson: string | null; + toJson: string | null; + ccJson: string | null; + blobId: string | null; + hasAttachment: boolean; + keywordsJson: string; + mailboxIds: string[]; +} + +export interface MailboxRow { + jmapAccountId: string; + id: string; + name: string; + parentId: string | null; + role: string | null; + sortOrder: number | null; + totalEmails: number | null; + unreadEmails: number | null; + totalThreads: number | null; + unreadThreads: number | null; + myRightsJson: string | null; + isSubscribed: boolean; +} diff --git a/playwright.integration-electron.config.ts b/playwright.integration-electron.config.ts index 7a0a7b2f..557f9507 100644 --- a/playwright.integration-electron.config.ts +++ b/playwright.integration-electron.config.ts @@ -22,10 +22,14 @@ import { defineConfig } from '@playwright/test'; export default defineConfig({ testDir: './integration/tests', // 11 asserts the native notification bridge fires from a real push; 12 - // asserts a real delivery reaches the encrypted local search index. 12 runs - // the REAL standalone-server boot (no ELECTRON_LOAD_URL), because that boot - // is what wires the index's store directory and its fd-3 key channel. - testMatch: /1[12]-electron-.*\.spec\.ts/, + // asserts a real delivery reaches the encrypted local search index; 13 asserts + // the offline mail REPLICA still serves a synced message's full body with the + // backend severed at the socket level. 12 and 13 run the REAL standalone-server + // boot (no ELECTRON_LOAD_URL), because that boot is what wires the store + // directory and the fd-3 key channel. + testMatch: /1[123]-electron-.*\.spec\.ts/, + // 13 syncs a real mailbox and then chains cycles, so it needs more than 90s; + // it sets its own per-test timeout, and this is the floor for the others. timeout: 90_000, expect: { timeout: 20_000 }, fullyParallel: false, diff --git a/playwright.integration.config.ts b/playwright.integration.config.ts index 0aeb9979..baa19f5c 100644 --- a/playwright.integration.config.ts +++ b/playwright.integration.config.ts @@ -30,7 +30,11 @@ export default defineConfig({ // Playwright image) has no Electron binary compatible with that // container's platform, so they must never be swept in by this config's // default testDir glob. - testIgnore: ['11-electron-notification.spec.ts', '12-electron-mail-index.spec.ts'], + testIgnore: [ + '11-electron-notification.spec.ts', + '12-electron-mail-index.spec.ts', + '13-electron-offline-replica.spec.ts', + ], // next dev compiles routes lazily and each test logs in fresh, so give // individual tests and their polling assertions generous headroom. timeout: 90_000, diff --git a/stores/auth-store.ts b/stores/auth-store.ts index f302a0cc..6e11de84 100644 --- a/stores/auth-store.ts +++ b/stores/auth-store.ts @@ -1,6 +1,7 @@ import { create } from 'zustand'; import { persist } from 'zustand/middleware'; import { JMAPClient, RateLimitError } from '@/lib/jmap/client'; +import { withOfflineFallback } from '@/lib/offline-fallback-client'; import type { IJMAPClient } from '@/lib/jmap/client-interface'; import { useIdentityStore } from './identity-store'; import { setClientLookup } from './client-registry'; @@ -638,14 +639,14 @@ export const useAuthStore = create()( } else { // Legacy fallback for pre-0.16 Stalwart, which accepts the TOTP // appended to the password over basic auth. - client = new JMAPClient(serverUrl, username, `${password}$${totp}`); + client = withOfflineFallback(new JMAPClient(serverUrl, username, `${password}$${totp}`)); await client.connect(); const { useTotpReauthStore } = await import('@/stores/totp-reauth-store'); client.enableTotpReauth(password, () => useTotpReauthStore.getState().requestTotp()); debug.log('auth', 'TOTP re-auth enabled (legacy basic-auth path)'); } } else { - client = new JMAPClient(serverUrl, username, password); + client = withOfflineFallback(new JMAPClient(serverUrl, username, password)); await client.connect(); } @@ -1426,7 +1427,7 @@ export const useAuthStore = create()( const res = await apiFetch(`/api/auth/session?slot=${targetAccount.cookieSlot}`, { method: 'PUT' }); if (res.ok) { const { serverUrl, username, password } = await res.json(); - targetClient = new JMAPClient(serverUrl, username, password); + targetClient = withOfflineFallback(new JMAPClient(serverUrl, username, password)); bindClientStatusHandlers(targetClient, set, get, accountId); await targetClient.connect(); clients.set(accountId, targetClient); @@ -1687,7 +1688,7 @@ export const useAuthStore = create()( const res = await apiFetch(`/api/auth/session?slot=${account.cookieSlot}`, { method: 'PUT' }); if (res.ok) { const { serverUrl, username, password } = await res.json(); - const client = new JMAPClient(serverUrl, username, password); + const client = withOfflineFallback(new JMAPClient(serverUrl, username, password)); bindClientStatusHandlers(client, set, get, account.id); await client.connect(); clients.set(account.id, client); @@ -1897,7 +1898,7 @@ export const useAuthStore = create()( throw new Error('Incomplete session data'); } const { serverUrl, username, password } = data; - const client = new JMAPClient(serverUrl, username, password); + const client = withOfflineFallback(new JMAPClient(serverUrl, username, password)); await client.connect(); const accountId = generateAccountId(username, serverUrl); diff --git a/stores/email-store.ts b/stores/email-store.ts index f0b97c0e..3b0359bb 100644 --- a/stores/email-store.ts +++ b/stores/email-store.ts @@ -2847,6 +2847,22 @@ export const useEmailStore = create((set, get) => ({ // arrived, i.e. index everything except the delivery that triggered it. const scheduleIndexUpdate = () => { void (async () => { + // The offline REPLICA (lib/offline-replica/**) rides the same trigger. + // It is a SEPARATE subsystem from the search index: the index keeps a + // plain-text excerpt for retrieval, the replica keeps full bodies plus + // properly-provenanced /changes cursors so mail stays READABLE with no + // network. Both write the same encrypted file on separate connections, + // and both are request-scoped with no background worker. Unlike the + // index, the replica DOES care about Mailbox changes - it holds the + // folder counters. + try { + const { syncOnStateChange } = await import('@/lib/offline-replica-client'); + syncOnStateChange(change, { + slot: useAccountStore.getState().getActiveAccount()?.cookieSlot, + }); + } catch { + /* the replica is optional; never let it affect mail handling */ + } try { const { indexOnStateChange } = await import('@/lib/mail-index-client'); const mailIds = get().emails.slice(0, 100).map((e) => e.id);