Merge branch 'claude/webmail-offline-replica' into 'dev'
feat(electron): real offline mail replica — delta sync, full bodies, retention See merge request gitlab-instance-b9b5cf2f/vncmail-plus!6
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
// The two-part fallback gate.
|
||||
//
|
||||
// `lib/jmap/client.ts`'s read methods swallow their own errors and return
|
||||
// plausible success, so a "looks empty" result is NOT evidence of a network
|
||||
// failure - it is also what a genuinely empty folder returns, and
|
||||
// `getMailboxes()` fabricates a synthetic Inbox rather than throwing. Falling back
|
||||
// on the shape alone would serve stale replica rows over a folder the user had
|
||||
// just emptied. So the gate is: suspicious result AND a `fetch` rejection recorded
|
||||
// during that exact call.
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { IJMAPClient } from '@/lib/jmap/client-interface';
|
||||
import type { Email, Mailbox } from '@/lib/jmap/types';
|
||||
import { noteTransportFailure, resetTransportHealth } from '@/lib/jmap/transport-health';
|
||||
|
||||
const readOfflineMailboxes = vi.fn();
|
||||
const readOfflineList = vi.fn();
|
||||
const readOfflineMessage = vi.fn();
|
||||
const isReplicaUnavailable = vi.fn(() => false);
|
||||
|
||||
vi.mock('@/lib/offline-replica-client', () => ({
|
||||
readOfflineMailboxes: (...a: unknown[]) => readOfflineMailboxes(...a),
|
||||
readOfflineList: (...a: unknown[]) => readOfflineList(...a),
|
||||
readOfflineMessage: (...a: unknown[]) => readOfflineMessage(...a),
|
||||
isReplicaUnavailable: () => isReplicaUnavailable(),
|
||||
}));
|
||||
|
||||
vi.mock('@/stores/account-store', () => ({
|
||||
useAccountStore: {
|
||||
getState: () => ({
|
||||
accounts: [{ id: 'alice@mail.example.org', cookieSlot: 3, serverIdentifiers: [] }],
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const { withOfflineFallback } = await import('@/lib/offline-fallback-client');
|
||||
|
||||
function replicaEmail(id: string): Email {
|
||||
return {
|
||||
id, threadId: 't', mailboxIds: { inbox: true }, keywords: {}, size: 1,
|
||||
receivedAt: '2026-08-01T00:00:00.000Z', hasAttachment: false,
|
||||
htmlBody: [{ partId: '1', blobId: 'b', size: 1, type: 'text/html' }],
|
||||
bodyValues: { '1': { value: '<p>from the replica</p>' } },
|
||||
};
|
||||
}
|
||||
|
||||
interface Stub extends Partial<IJMAPClient> {
|
||||
getEmail: IJMAPClient['getEmail'];
|
||||
getEmails: IJMAPClient['getEmails'];
|
||||
getMailboxes: IJMAPClient['getMailboxes'];
|
||||
getAllMailboxes: IJMAPClient['getAllMailboxes'];
|
||||
}
|
||||
|
||||
/** Reproduces the client's real error-swallowing shapes. */
|
||||
function stubClient(overrides: Partial<Stub> = {}): IJMAPClient {
|
||||
const stub = {
|
||||
getUsername: () => 'alice',
|
||||
getServerUrl: () => 'https://mail.example.org',
|
||||
getAccountId: () => 'primary',
|
||||
getEmail: async () => null,
|
||||
getEmails: async () => ({ emails: [] as Email[], hasMore: false, total: 0 }),
|
||||
getMailboxes: async () => ([
|
||||
// The exact placeholder client.ts fabricates on failure.
|
||||
{ id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0,
|
||||
unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true,
|
||||
myRights: {} } as unknown as Mailbox,
|
||||
]),
|
||||
getAllMailboxes: async () => ([
|
||||
{ id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0,
|
||||
unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true,
|
||||
myRights: {} } as unknown as Mailbox,
|
||||
]),
|
||||
...overrides,
|
||||
};
|
||||
return stub as unknown as IJMAPClient;
|
||||
}
|
||||
|
||||
describe('withOfflineFallback', () => {
|
||||
beforeEach(() => {
|
||||
resetTransportHealth();
|
||||
vi.clearAllMocks();
|
||||
isReplicaUnavailable.mockReturnValue(false);
|
||||
});
|
||||
|
||||
it('does NOT consult the replica when the server answered "empty"', async () => {
|
||||
// The whole point. An empty folder must render empty, not as whatever the
|
||||
// replica last held.
|
||||
const client = withOfflineFallback(stubClient());
|
||||
const result = await client.getEmails('inbox');
|
||||
expect(result.emails).toEqual([]);
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
|
||||
expect(await client.getEmail('e1')).toBeNull();
|
||||
expect(readOfflineMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('consults the replica when a transport failure happened DURING the call', async () => {
|
||||
readOfflineList.mockResolvedValue({
|
||||
emails: [replicaEmail('e1')], total: 1, hasMore: false,
|
||||
});
|
||||
const client = withOfflineFallback(
|
||||
stubClient({
|
||||
getEmails: async () => {
|
||||
// What authenticatedFetch does when `fetch` rejects.
|
||||
noteTransportFailure();
|
||||
return { emails: [], hasMore: false, total: 0 };
|
||||
},
|
||||
}),
|
||||
);
|
||||
const result = await client.getEmails('inbox', undefined, 25, 0);
|
||||
expect(result.emails.map((e) => e.id)).toEqual(['e1']);
|
||||
expect(result.total).toBe(1);
|
||||
// And it asks for the right slot, so a multi-account shell reads the right file.
|
||||
expect(readOfflineList).toHaveBeenCalledWith('inbox', { limit: 25, offset: 0, slot: 3 });
|
||||
});
|
||||
|
||||
it('ignores a stale transport failure from BEFORE the call', async () => {
|
||||
// The counter is sampled per call precisely so an old failure cannot make a
|
||||
// later successful-but-empty read look offline.
|
||||
noteTransportFailure();
|
||||
const client = withOfflineFallback(stubClient());
|
||||
await client.getEmails('inbox');
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('serves a full message from the replica, but refuses an envelope-only hit', async () => {
|
||||
// An envelope with no bodyValues would render blank AND leave the viewer's
|
||||
// isBodyLoading gate stuck on its skeleton, which is worse than saying the
|
||||
// message is unavailable.
|
||||
readOfflineMessage.mockResolvedValue({ email: replicaEmail('e1'), hasBody: true });
|
||||
const client = withOfflineFallback(
|
||||
stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }),
|
||||
);
|
||||
const email = await client.getEmail('e1');
|
||||
expect(email?.bodyValues?.['1'].value).toContain('from the replica');
|
||||
|
||||
resetTransportHealth();
|
||||
readOfflineMessage.mockResolvedValue({ email: replicaEmail('e2'), hasBody: false });
|
||||
const client2 = withOfflineFallback(
|
||||
stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }),
|
||||
);
|
||||
expect(await client2.getEmail('e2')).toBeNull();
|
||||
});
|
||||
|
||||
it('recognises the synthetic Inbox placeholder and replaces it', async () => {
|
||||
readOfflineMailboxes.mockResolvedValue([
|
||||
{ id: 'mb1', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 9, unreadEmails: 2,
|
||||
totalThreads: 9, unreadThreads: 2, isSubscribed: true, myRights: {} } as unknown as Mailbox,
|
||||
]);
|
||||
const client = withOfflineFallback(
|
||||
stubClient({
|
||||
getAllMailboxes: async () => {
|
||||
noteTransportFailure();
|
||||
return [
|
||||
{ id: 'INBOX', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 0,
|
||||
unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true,
|
||||
myRights: {} } as unknown as Mailbox,
|
||||
];
|
||||
},
|
||||
}),
|
||||
);
|
||||
const mailboxes = await client.getAllMailboxes();
|
||||
expect(mailboxes.map((m) => m.id)).toEqual(['mb1']);
|
||||
});
|
||||
|
||||
it('keeps a REAL single-mailbox server result even after a transport failure', async () => {
|
||||
// A genuine server that happens to return one inbox has a real id and real
|
||||
// counts; only the exact placeholder shape may be replaced.
|
||||
const real = {
|
||||
id: 'real-inbox-id', name: 'Inbox', role: 'inbox', sortOrder: 0, totalEmails: 12,
|
||||
unreadEmails: 1, totalThreads: 12, unreadThreads: 1, isSubscribed: true, myRights: {},
|
||||
} as unknown as Mailbox;
|
||||
const client = withOfflineFallback(
|
||||
stubClient({ getAllMailboxes: async () => { noteTransportFailure(); return [real]; } }),
|
||||
);
|
||||
expect((await client.getAllMailboxes())[0].id).toBe('real-inbox-id');
|
||||
expect(readOfflineMailboxes).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('never answers a read scoped to a delegated account', async () => {
|
||||
// v1 replicates the PRIMARY mail account only, so the replica has no rows for
|
||||
// a shared account and answering "empty" would be worse than the client's own.
|
||||
const client = withOfflineFallback(
|
||||
stubClient({
|
||||
getEmails: async () => { noteTransportFailure(); return { emails: [], hasMore: false, total: 0 }; },
|
||||
}),
|
||||
);
|
||||
await client.getEmails('inbox', 'someone-elses-account');
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('never answers a keyword- or category-filtered query', async () => {
|
||||
// Those are server-side queries the replica does not reproduce. Serving an
|
||||
// unfiltered page in their place would silently show the wrong set.
|
||||
const failing = async () => { noteTransportFailure(); return { emails: [], hasMore: false, total: 0 }; };
|
||||
const c1 = withOfflineFallback(stubClient({ getEmails: failing }));
|
||||
await c1.getEmails('inbox', undefined, 25, 0, '$flagged');
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
|
||||
resetTransportHealth();
|
||||
const c2 = withOfflineFallback(stubClient({ getEmails: failing }));
|
||||
await c2.getEmails('inbox', undefined, 25, 0, undefined, true, { from: 'x' });
|
||||
expect(readOfflineList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('stops asking once the replica reports itself absent', async () => {
|
||||
isReplicaUnavailable.mockReturnValue(true);
|
||||
const client = withOfflineFallback(
|
||||
stubClient({ getEmail: async () => { noteTransportFailure(); return null; } }),
|
||||
);
|
||||
expect(await client.getEmail('e1')).toBeNull();
|
||||
expect(readOfflineMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is idempotent, so re-wrapping a client does not stack fallbacks', async () => {
|
||||
readOfflineMessage.mockResolvedValue({ email: replicaEmail('e1'), hasBody: true });
|
||||
const base = stubClient({ getEmail: async () => { noteTransportFailure(); return null; } });
|
||||
const once = withOfflineFallback(base);
|
||||
const twice = withOfflineFallback(once);
|
||||
expect(twice).toBe(once);
|
||||
await twice.getEmail('e1');
|
||||
expect(readOfflineMessage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
+15
-1
@@ -3,6 +3,7 @@ import type { SieveScript, SieveCapabilities } from "./sieve-types";
|
||||
import type { IJMAPClient } from "./client-interface";
|
||||
import { toWildcardQuery } from "./search-utils";
|
||||
import { batched, itemsPerRequest } from "./request-limits";
|
||||
import { noteTransportFailure, noteTransportSuccess } from "./transport-health";
|
||||
import { debug } from "@/lib/debug";
|
||||
import { normalizeCalendarEventLike } from "@/lib/calendar-event-normalization";
|
||||
|
||||
@@ -683,11 +684,24 @@ export class JMAPClient implements IJMAPClient {
|
||||
try {
|
||||
response = await fetch(url, { ...init, headers });
|
||||
} catch (error) {
|
||||
// A `fetch` REJECTION - and only that - is a transport failure. Recorded so
|
||||
// the offline replica's read fallback can tell "the network is down" from
|
||||
// "the folder is empty", which the error-swallowing in getEmails/getEmail/
|
||||
// getMailboxes otherwise makes indistinguishable (see
|
||||
// lib/jmap/transport-health.ts). Deliberately NOT recorded for a 4xx/5xx or
|
||||
// a 429: in those cases the server answered, so it is reachable.
|
||||
noteTransportFailure();
|
||||
// Network error: retry once after brief delay (transient proxy/connection issues)
|
||||
if (this.reconnecting) throw error;
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
response = await fetch(url, { ...init, headers });
|
||||
try {
|
||||
response = await fetch(url, { ...init, headers });
|
||||
} catch (retryError) {
|
||||
noteTransportFailure();
|
||||
throw retryError;
|
||||
}
|
||||
}
|
||||
noteTransportSuccess();
|
||||
|
||||
// Handle 429 rate limiting - stop immediately, do not retry
|
||||
if (response.status === 429) {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// A single monotonic counter of JMAP TRANSPORT failures.
|
||||
//
|
||||
// WHY THIS EXISTS. The offline replica is a read-path FALLBACK, and to be one it
|
||||
// has to know that a read genuinely failed. `lib/jmap/client.ts` makes that
|
||||
// impossible to see from the outside: its read methods swallow their own errors
|
||||
// and return plausible-looking success. `getEmails()` returns
|
||||
// `{ emails: [], hasMore: false, total: 0 }`, so a dead network is
|
||||
// indistinguishable from an empty folder. `getEmail()` returns `null`.
|
||||
// `getMailboxes()` returns a SYNTHETIC single Inbox. Falling back on those shapes
|
||||
// alone would mean serving stale replica rows for a folder the user had genuinely
|
||||
// just emptied.
|
||||
//
|
||||
// So `authenticatedFetch` bumps this counter when, and only when, `fetch` itself
|
||||
// rejects - not on a 4xx, not on a 429 (that is a rate limit, and the server is
|
||||
// plainly reachable), not on a JMAP method error. The fallback layer samples the
|
||||
// counter before and after a call: a suspicious result PLUS an increment during
|
||||
// that exact call is a transport failure. Either signal alone is not enough.
|
||||
//
|
||||
// Module-level rather than per-client on purpose: it answers "is the network
|
||||
// working right now", which is a property of the machine, not of one account's
|
||||
// client instance.
|
||||
|
||||
let failures = 0;
|
||||
let lastFailureAt = 0;
|
||||
let lastSuccessAt = 0;
|
||||
|
||||
/** Called only when `fetch` itself rejects. Never for an HTTP status. */
|
||||
export function noteTransportFailure(): void {
|
||||
failures++;
|
||||
lastFailureAt = Date.now();
|
||||
}
|
||||
|
||||
export function noteTransportSuccess(): void {
|
||||
lastSuccessAt = Date.now();
|
||||
}
|
||||
|
||||
/** Monotonic. Sample before and after a call to attribute a failure to it. */
|
||||
export function transportFailureCount(): number {
|
||||
return failures;
|
||||
}
|
||||
|
||||
export function transportHealth(): {
|
||||
failures: number;
|
||||
lastFailureAt: number;
|
||||
lastSuccessAt: number;
|
||||
/** Best-effort "probably offline": a failure more recent than any success. */
|
||||
likelyOffline: boolean;
|
||||
} {
|
||||
return {
|
||||
failures,
|
||||
lastFailureAt,
|
||||
lastSuccessAt,
|
||||
likelyOffline: lastFailureAt > lastSuccessAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Test-only reset. */
|
||||
export function resetTransportHealth(): void {
|
||||
failures = 0;
|
||||
lastFailureAt = 0;
|
||||
lastSuccessAt = 0;
|
||||
}
|
||||
@@ -178,6 +178,12 @@ export class MailIndex {
|
||||
try {
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
// The offline replica (lib/offline-replica/**) is a SECOND connection to
|
||||
// this same file, writing disjoint tables. WAL lets a writer and readers
|
||||
// coexist, but two WRITERS get SQLITE_BUSY immediately without this - and
|
||||
// both subsystems are driven by the same renderer push handler, so they
|
||||
// genuinely do overlap.
|
||||
db.pragma('busy_timeout = 8000');
|
||||
version = readSchemaVersion(db);
|
||||
} catch {
|
||||
db.close();
|
||||
@@ -189,6 +195,12 @@ export class MailIndex {
|
||||
assertEncrypted(db, dbPath);
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
// The offline replica (lib/offline-replica/**) is a SECOND connection to
|
||||
// this same file, writing disjoint tables. WAL lets a writer and readers
|
||||
// coexist, but two WRITERS get SQLITE_BUSY immediately without this - and
|
||||
// both subsystems are driven by the same renderer push handler, so they
|
||||
// genuinely do overlap.
|
||||
db.pragma('busy_timeout = 8000');
|
||||
version = null;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
// The read-path fallback. Wraps an `IJMAPClient` so that when a mail read fails
|
||||
// because the network is down, the answer comes from the encrypted offline replica
|
||||
// instead of an empty list.
|
||||
//
|
||||
// WHY THIS SHAPE, AND NOT A CACHE. The replica is consulted ONLY after a read has
|
||||
// genuinely failed at the transport level. That ordering is the whole coherence
|
||||
// story for the design review's H3: the webmail does local delta arithmetic on
|
||||
// mailbox unread counts for mark-read/move/delete, and if the replica sat in FRONT
|
||||
// of the server that arithmetic would operate on replica numbers and need
|
||||
// reconciliation rules. Behind the server, an online session never sees a replica
|
||||
// value at all, and while offline any count drift is bounded and repaired by the
|
||||
// next `Mailbox/changes`.
|
||||
//
|
||||
// WHY IT IS NOT ENOUGH TO LOOK AT THE RESULT. `lib/jmap/client.ts`'s read methods
|
||||
// swallow their own errors and return plausible success: `getEmails()` returns an
|
||||
// empty page, `getEmail()` returns `null`, `getMailboxes()` returns a SYNTHETIC
|
||||
// single Inbox. Falling back on those shapes alone would serve stale replica rows
|
||||
// for a folder the user had genuinely just emptied. So the test is TWO-PART: a
|
||||
// suspicious result AND a `fetch` rejection recorded during that exact call
|
||||
// (`lib/jmap/transport-health.ts`). A 4xx, a 429 or a JMAP method error all mean
|
||||
// the server answered, so none of them triggers a fallback.
|
||||
//
|
||||
// Mutates the instance rather than wrapping it in a Proxy: `JMAPClient` is a large
|
||||
// class whose methods call each other through `this`, and instance patching keeps
|
||||
// `this` identity exactly as it was. Idempotent, so re-wrapping the same client is
|
||||
// harmless.
|
||||
|
||||
import { generateAccountId } from '@/lib/account-utils';
|
||||
import type { IJMAPClient } from '@/lib/jmap/client-interface';
|
||||
import type { Email, Mailbox } from '@/lib/jmap/types';
|
||||
import { transportFailureCount } from '@/lib/jmap/transport-health';
|
||||
import {
|
||||
isReplicaUnavailable, readOfflineList, readOfflineMailboxes, readOfflineMessage,
|
||||
} from '@/lib/offline-replica-client';
|
||||
|
||||
const WRAPPED = Symbol.for('vncmail.offlineFallback.wrapped');
|
||||
|
||||
/**
|
||||
* A `getMailboxes()` result that is really the client's offline placeholder.
|
||||
*
|
||||
* `client.ts` fabricates exactly this on failure: one mailbox, id `INBOX`, role
|
||||
* `inbox`, zero counts. Matching it precisely matters - a real server that happens
|
||||
* to return a single inbox has a real id and real counts.
|
||||
*/
|
||||
function isSyntheticMailboxList(mailboxes: readonly Mailbox[]): boolean {
|
||||
return (
|
||||
mailboxes.length === 1 &&
|
||||
mailboxes[0]?.id === 'INBOX' &&
|
||||
mailboxes[0]?.totalEmails === 0 &&
|
||||
mailboxes[0]?.unreadEmails === 0
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolves this client's cookie slot, so a multi-account shell reads the right replica. */
|
||||
async function slotFor(client: IJMAPClient): Promise<number | undefined> {
|
||||
try {
|
||||
const { useAccountStore } = await import('@/stores/account-store');
|
||||
const id = generateAccountId(client.getUsername(), client.getServerUrl());
|
||||
const accounts = useAccountStore.getState().accounts;
|
||||
const match =
|
||||
accounts.find((a) => a.id === id) ??
|
||||
accounts.find((a) => a.serverIdentifiers?.includes(id));
|
||||
return match?.cookieSlot;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the replica may answer for this call.
|
||||
*
|
||||
* v1 replicates the PRIMARY mail account only, so a read explicitly scoped to a
|
||||
* delegated/shared account must never be answered from it - the replica simply has
|
||||
* no rows, and answering "empty" would be worse than the client's own empty.
|
||||
*/
|
||||
function scopedToPrimary(client: IJMAPClient, accountId?: string): boolean {
|
||||
if (!accountId) return true;
|
||||
try {
|
||||
return accountId === client.getAccountId();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function withOfflineFallback<T extends IJMAPClient>(client: T): T {
|
||||
const flagged = client as unknown as Record<symbol, boolean | undefined>;
|
||||
if (flagged[WRAPPED]) return client;
|
||||
flagged[WRAPPED] = true;
|
||||
|
||||
const target = client as unknown as IJMAPClient;
|
||||
const originalGetEmail = target.getEmail.bind(target);
|
||||
const originalGetEmails = target.getEmails.bind(target);
|
||||
const originalGetMailboxes = target.getMailboxes.bind(target);
|
||||
const originalGetAllMailboxes = target.getAllMailboxes.bind(target);
|
||||
|
||||
target.getEmail = async (emailId: string, accountId?: string): Promise<Email | null> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetEmail(emailId, accountId);
|
||||
if (online) return online;
|
||||
if (isReplicaUnavailable()) return online;
|
||||
// `null` alone is ambiguous: it is also what a genuinely-missing id returns.
|
||||
// Only a transport failure during THIS call earns a fallback.
|
||||
if (transportFailureCount() === before) return online;
|
||||
if (!scopedToPrimary(client, accountId)) return online;
|
||||
|
||||
const offline = await readOfflineMessage(emailId, await slotFor(client));
|
||||
// An envelope with no body would render blank AND leave the viewer's
|
||||
// `isBodyLoading` gate stuck, so it is not an answer - better to keep the
|
||||
// client's `null` and let the UI say the message is unavailable offline.
|
||||
if (!offline?.email || !offline.hasBody) return online;
|
||||
return offline.email;
|
||||
};
|
||||
|
||||
target.getEmails = async (
|
||||
mailboxId?: string,
|
||||
accountId?: string,
|
||||
limit: number = 50,
|
||||
position: number = 0,
|
||||
hasKeyword?: string,
|
||||
pinnedFirst?: boolean,
|
||||
extraFilter?: Record<string, unknown>,
|
||||
): Promise<{ emails: Email[]; hasMore: boolean; total: number }> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetEmails(
|
||||
mailboxId, accountId, limit, position, hasKeyword, pinnedFirst, extraFilter,
|
||||
);
|
||||
if (online.emails.length > 0) return online;
|
||||
if (isReplicaUnavailable()) return online;
|
||||
if (transportFailureCount() === before) return online;
|
||||
if (!scopedToPrimary(client, accountId)) return online;
|
||||
// A keyword or category filter is a server-side query the replica does not
|
||||
// reproduce. Serving an unfiltered page in its place would silently show the
|
||||
// wrong set, which is worse than showing nothing.
|
||||
if (hasKeyword || extraFilter) return online;
|
||||
|
||||
const offline = await readOfflineList(mailboxId ?? null, {
|
||||
limit,
|
||||
offset: position,
|
||||
slot: await slotFor(client),
|
||||
});
|
||||
if (!offline || offline.emails.length === 0) return online;
|
||||
return { emails: offline.emails, hasMore: offline.hasMore, total: offline.total };
|
||||
};
|
||||
|
||||
const mailboxFallback = async (
|
||||
online: Mailbox[],
|
||||
before: number,
|
||||
accountId?: string,
|
||||
): Promise<Mailbox[]> => {
|
||||
// Bail out unless the result is EMPTY or is the exact synthetic placeholder.
|
||||
// Testing `length > 1` here was a real bug found by
|
||||
// `lib/__tests__/offline-fallback-client.test.ts`: a server that legitimately
|
||||
// exposes a single mailbox got its real folder - real id, real counts -
|
||||
// replaced by replica rows the moment any unrelated transport blip was
|
||||
// recorded during the call.
|
||||
if (online.length > 0 && !isSyntheticMailboxList(online)) return online;
|
||||
if (isReplicaUnavailable()) return online;
|
||||
if (transportFailureCount() === before) return online;
|
||||
if (!scopedToPrimary(client, accountId)) return online;
|
||||
const offline = await readOfflineMailboxes(await slotFor(client));
|
||||
if (!offline || offline.length === 0) return online;
|
||||
return offline;
|
||||
};
|
||||
|
||||
target.getMailboxes = async (accountId?: string): Promise<Mailbox[]> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetMailboxes(accountId);
|
||||
return mailboxFallback(online, before, accountId);
|
||||
};
|
||||
|
||||
target.getAllMailboxes = async (): Promise<Mailbox[]> => {
|
||||
const before = transportFailureCount();
|
||||
const online = await originalGetAllMailboxes();
|
||||
// `getAllMailboxes` falls back internally to `getMailboxes()`, so an offline
|
||||
// run arrives here as the synthetic single Inbox rather than an empty list.
|
||||
return mailboxFallback(online, before);
|
||||
};
|
||||
|
||||
return client;
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
// Renderer-side client for the offline mail replica.
|
||||
//
|
||||
// The replica is EVENT-DRIVEN, exactly like the search index next to it: the
|
||||
// renderer already holds the live JMAP push connection, so a `StateChange` is what
|
||||
// triggers a sync cycle. There is no polling loop and no background worker.
|
||||
//
|
||||
// One cycle is BOUNDED (see lib/offline-replica/sync.ts's BUDGET), so a first
|
||||
// sync of a large mailbox needs several. `unfinishedWork` is the server saying
|
||||
// "call again", and `chainSync` below does that with a hard cap - the cap matters,
|
||||
// because an "unfinished work" signal that is true for a condition the cycle
|
||||
// cannot change is how the mobile client ended up chaining a new cycle every five
|
||||
// seconds forever.
|
||||
//
|
||||
// Every function here is best-effort and never throws: offline storage failing to
|
||||
// update must never break the mail UI.
|
||||
|
||||
import { apiFetch } from '@/lib/browser-navigation';
|
||||
import { debug } from '@/lib/debug';
|
||||
import type { Email, Mailbox } from '@/lib/jmap/types';
|
||||
import type { StateChange } from '@/lib/jmap/types';
|
||||
|
||||
export interface RetentionPolicy {
|
||||
envelopeDays: number;
|
||||
bodyDays: number;
|
||||
maxBodyMB: number;
|
||||
}
|
||||
|
||||
export interface CycleReport {
|
||||
ok: boolean;
|
||||
unfinishedWork: boolean;
|
||||
bootstrapped: boolean;
|
||||
reconciled: boolean;
|
||||
mailboxesWritten: number;
|
||||
envelopesWritten: number;
|
||||
envelopesDeleted: number;
|
||||
bodiesWritten: number;
|
||||
bodiesEvicted: number;
|
||||
coveragePhase: string;
|
||||
resyncRequired: boolean;
|
||||
warnings: string[];
|
||||
errorClass?: string;
|
||||
error?: string;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export interface ReplicaStats {
|
||||
mailboxes: number;
|
||||
envelopes: number;
|
||||
bodies: number;
|
||||
bodyBytes: number;
|
||||
wantedBodies: number;
|
||||
giveUps: number;
|
||||
newest: string | null;
|
||||
oldest: string | null;
|
||||
fileBytes: number;
|
||||
}
|
||||
|
||||
export interface ReplicaStatus {
|
||||
ok: boolean;
|
||||
policy: RetentionPolicy;
|
||||
limits: Record<keyof RetentionPolicy, { min: number; max: number }>;
|
||||
synced: boolean;
|
||||
stats: ReplicaStats | null;
|
||||
coveragePhase: string;
|
||||
coveredFrom?: string | null;
|
||||
resyncRequired: boolean;
|
||||
lastCycleAt: number | null;
|
||||
lastCycleOk: boolean | null;
|
||||
lastCycleError?: string | null;
|
||||
}
|
||||
|
||||
/** Set once the server says the feature isn't there, so we stop asking. */
|
||||
let knownUnavailable = false;
|
||||
let inFlight: Promise<CycleReport | null> | null = null;
|
||||
|
||||
function slotQuery(slot?: number, extra?: string): string {
|
||||
const params = new URLSearchParams();
|
||||
if (typeof slot === 'number') params.set('slot', String(slot));
|
||||
const base = params.toString();
|
||||
if (extra && base) return `?${base}&${extra}`;
|
||||
if (extra) return `?${extra}`;
|
||||
return base ? `?${base}` : '';
|
||||
}
|
||||
|
||||
/** True when the replica is known to be absent (not the desktop shell, or gated off). */
|
||||
export function isReplicaUnavailable(): boolean {
|
||||
return knownUnavailable;
|
||||
}
|
||||
|
||||
export function resetReplicaAvailability(): void {
|
||||
knownUnavailable = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs ONE cycle. Single-flighted on the renderer as well as the server, so a
|
||||
* burst of deliveries coalesces instead of queueing N overlapping requests that
|
||||
* the server would then serialise anyway.
|
||||
*/
|
||||
export async function syncOnce(
|
||||
opts: { slot?: number; policy?: RetentionPolicy; forceResync?: boolean } = {},
|
||||
): Promise<CycleReport | null> {
|
||||
if (knownUnavailable) return null;
|
||||
if (inFlight) return inFlight;
|
||||
|
||||
const run = (async (): Promise<CycleReport | null> => {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/sync${slotQuery(opts.slot)}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ policy: opts.policy, forceResync: opts.forceResync === true }),
|
||||
});
|
||||
// 404 = not the desktop shell. Permanent for this page load; stop asking so
|
||||
// a busy mailbox doesn't post per delivery.
|
||||
if (response.status === 404) { knownUnavailable = true; return null; }
|
||||
if (response.status === 503) {
|
||||
const body = await response.json().catch(() => ({}));
|
||||
// A transport-class 503 means the BACKEND is unreachable, which is normal
|
||||
// and temporary - it must not latch the feature off for the session. Only
|
||||
// a missing binding / key channel does that.
|
||||
const code = typeof body?.code === 'string' ? body.code : '';
|
||||
if (code === 'no-binding' || code === 'no-key-channel' || code === 'unavailable') {
|
||||
knownUnavailable = true;
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) return null;
|
||||
const body = await response.json();
|
||||
debug.log('push', '[replica] cycle', body?.report);
|
||||
return (body?.report ?? null) as CycleReport | null;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
inFlight = null;
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
/** Hard cap on chained cycles per trigger. */
|
||||
export const MAX_CHAINED_CYCLES = 12;
|
||||
|
||||
/**
|
||||
* Runs cycles while the server reports unfinished work.
|
||||
*
|
||||
* The cap is the whole point. `unfinishedWork` is a hint, and a hint that stays
|
||||
* true for something the cycle cannot resolve turns into an endless chain - which
|
||||
* is exactly what happened on the mobile client when a body-queue counter reported
|
||||
* attempted rather than inserted rows. The server-side fixes make that
|
||||
* self-terminating; this cap means even a future regression costs a bounded number
|
||||
* of requests rather than an infinite loop.
|
||||
*/
|
||||
export async function chainSync(
|
||||
opts: { slot?: number; max?: number; onReport?: (report: CycleReport) => void } = {},
|
||||
): Promise<CycleReport | null> {
|
||||
const max = Math.min(opts.max ?? MAX_CHAINED_CYCLES, MAX_CHAINED_CYCLES);
|
||||
let last: CycleReport | null = null;
|
||||
for (let i = 0; i < max; i++) {
|
||||
const report = await syncOnce({ slot: opts.slot });
|
||||
if (!report) return last;
|
||||
last = report;
|
||||
opts.onReport?.(report);
|
||||
if (!report.ok || !report.unfinishedWork) return report;
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
/** The push-driven entry point. Fire-and-forget: the mail UI must not wait on it. */
|
||||
export function syncOnStateChange(change: StateChange, opts: { slot?: number } = {}): void {
|
||||
if (knownUnavailable) return;
|
||||
// Only mail-shaped changes are worth a cycle. A `Mailbox` state change alone is
|
||||
// usually just an unread-count move, but the replica DOES hold those counts, so
|
||||
// unlike the search index it is worth reacting to.
|
||||
const relevant = Object.values(change.changed ?? {}).some(
|
||||
(perAccount) => perAccount && (perAccount.Email || perAccount.Mailbox),
|
||||
);
|
||||
if (!relevant) return;
|
||||
void syncOnce({ slot: opts.slot });
|
||||
}
|
||||
|
||||
export async function fetchReplicaStatus(slot?: number): Promise<ReplicaStatus | null> {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`);
|
||||
if (!response.ok) return null;
|
||||
return (await response.json()) as ReplicaStatus;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateRetentionPolicy(
|
||||
policy: RetentionPolicy,
|
||||
slot?: number,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(policy),
|
||||
});
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function purgeReplica(slot?: number): Promise<boolean> {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/status${slotQuery(slot)}`, { method: 'DELETE' });
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── reads ───────────────────────────────────────────────────────────────────
|
||||
|
||||
interface ReadEnvelope<T> {
|
||||
ok?: boolean;
|
||||
available?: boolean;
|
||||
error?: string;
|
||||
data?: T;
|
||||
}
|
||||
|
||||
async function read<T>(query: string): Promise<(T & { available: boolean }) | null> {
|
||||
if (knownUnavailable) return null;
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/mail${query}`);
|
||||
if (response.status === 404) { knownUnavailable = true; return null; }
|
||||
if (!response.ok) return null;
|
||||
const body = (await response.json()) as ReadEnvelope<unknown> & Record<string, unknown>;
|
||||
if (body?.available !== true) return null;
|
||||
return body as unknown as T & { available: boolean };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readOfflineMailboxes(slot?: number): Promise<Mailbox[] | null> {
|
||||
const body = await read<{ mailboxes: Mailbox[] }>(slotQuery(slot, 'kind=mailboxes'));
|
||||
return body?.mailboxes ?? null;
|
||||
}
|
||||
|
||||
export async function readOfflineList(
|
||||
mailboxId: string | null,
|
||||
opts: { limit?: number; offset?: number; slot?: number } = {},
|
||||
): Promise<{ emails: Email[]; total: number; hasMore: boolean } | null> {
|
||||
const params = new URLSearchParams({ kind: 'list' });
|
||||
if (mailboxId !== null) params.set('mailboxId', mailboxId);
|
||||
if (opts.limit !== undefined) params.set('limit', String(opts.limit));
|
||||
if (opts.offset !== undefined) params.set('offset', String(opts.offset));
|
||||
const body = await read<{ emails: Email[]; total: number; hasMore: boolean }>(
|
||||
slotQuery(opts.slot, params.toString()),
|
||||
);
|
||||
if (!body) return null;
|
||||
return { emails: body.emails ?? [], total: body.total ?? 0, hasMore: body.hasMore === true };
|
||||
}
|
||||
|
||||
export async function readOfflineMessage(
|
||||
id: string,
|
||||
slot?: number,
|
||||
): Promise<{ email: Email | null; hasBody: boolean } | null> {
|
||||
const params = new URLSearchParams({ kind: 'message', id });
|
||||
const body = await read<{ email: Email | null; hasBody: boolean }>(
|
||||
slotQuery(slot, params.toString()),
|
||||
);
|
||||
if (!body) return null;
|
||||
return { email: body.email ?? null, hasBody: body.hasBody === true };
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
advanceOneMs, madeForwardProgress, normalisePage, pageIsEmpty, planEmailFetches,
|
||||
planMailboxFetches, updatedPropertiesAreCountsOnly, type ChangesPage,
|
||||
} from '../apply';
|
||||
import { asChangesState } from '../states';
|
||||
|
||||
function page(partial: Partial<ChangesPage>): ChangesPage {
|
||||
return {
|
||||
oldState: asChangesState('old'),
|
||||
newState: asChangesState('new'),
|
||||
hasMoreChanges: false,
|
||||
created: [],
|
||||
updated: [],
|
||||
destroyed: [],
|
||||
...partial,
|
||||
};
|
||||
}
|
||||
|
||||
describe('normalisePage', () => {
|
||||
it('lets a destroyed id win outright over created and updated', () => {
|
||||
// Fetching an id that is also destroyed spends a request to get `notFound`.
|
||||
const out = normalisePage(page({ created: ['a', 'b'], updated: ['a'], destroyed: ['a'] }));
|
||||
expect(out.created).toEqual(['b']);
|
||||
expect(out.updated).toEqual([]);
|
||||
expect(out.destroyed).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('treats an id in both created and updated as a create', () => {
|
||||
// The create path fetches the full envelope tier, which already contains the
|
||||
// updated values - so an extra 3-property fetch would be pure waste.
|
||||
const out = normalisePage(page({ created: ['a'], updated: ['a'] }));
|
||||
expect(out.created).toEqual(['a']);
|
||||
expect(out.updated).toEqual([]);
|
||||
});
|
||||
|
||||
it('deduplicates within each bucket', () => {
|
||||
const out = normalisePage(page({ created: ['a', 'a'], destroyed: ['b', 'b'] }));
|
||||
expect(out.created).toEqual(['a']);
|
||||
expect(out.destroyed).toEqual(['b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pageIsEmpty', () => {
|
||||
it('is true only when nothing changed', () => {
|
||||
// An empty page STILL has to advance the cursor: skipping it re-requests the
|
||||
// same position forever.
|
||||
expect(pageIsEmpty(page({}))).toBe(true);
|
||||
expect(pageIsEmpty(page({ updated: ['a'] }))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('planEmailFetches', () => {
|
||||
it('drops an updated id we do not hold locally, BEFORE any fetch is issued', () => {
|
||||
// The absent case is an unconditional no-op. Fetching it would need a
|
||||
// `receivedAt` the 3-property response cannot supply and the schema's
|
||||
// NOT NULL would reject. Coverage enumerates CURRENT state, so it will pick
|
||||
// the record up with the updated values anyway.
|
||||
const plan = planEmailFetches(page({ updated: ['have', 'missing'] }), new Set(['have']));
|
||||
expect(plan.updateIds).toEqual(['have']);
|
||||
});
|
||||
|
||||
it('keeps creates unconditional - presence is irrelevant for a create', () => {
|
||||
const plan = planEmailFetches(page({ created: ['new'] }), new Set());
|
||||
expect(plan.createIds).toEqual(['new']);
|
||||
});
|
||||
|
||||
it('never routes an id into both the create and the update fetch', () => {
|
||||
const plan = planEmailFetches(page({ created: ['a'], updated: ['a'] }), new Set(['a']));
|
||||
expect(plan.createIds).toEqual(['a']);
|
||||
expect(plan.updateIds).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updatedPropertiesAreCountsOnly', () => {
|
||||
it('is true for the four counters and for an empty list', () => {
|
||||
expect(updatedPropertiesAreCountsOnly(['unreadEmails'])).toBe(true);
|
||||
expect(updatedPropertiesAreCountsOnly(['totalEmails', 'unreadThreads'])).toBe(true);
|
||||
// "nothing but the state token moved" is counts-only vacuously.
|
||||
expect(updatedPropertiesAreCountsOnly([])).toBe(true);
|
||||
});
|
||||
|
||||
it('is false when the server will not say what changed', () => {
|
||||
// `null` means "assume everything", so the whole object must be re-fetched.
|
||||
expect(updatedPropertiesAreCountsOnly(null)).toBe(false);
|
||||
expect(updatedPropertiesAreCountsOnly(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it('is false as soon as one non-count property is present', () => {
|
||||
expect(updatedPropertiesAreCountsOnly(['unreadEmails', 'name'])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('planMailboxFetches', () => {
|
||||
it('routes updates to the cheap four-integer patch when only counts moved', () => {
|
||||
const plan = planMailboxFetches(
|
||||
page({ created: ['new'], updated: ['old'], updatedProperties: ['unreadEmails'] }),
|
||||
);
|
||||
expect(plan.fullIds).toEqual(['new']);
|
||||
expect(plan.countOnlyIds).toEqual(['old']);
|
||||
});
|
||||
|
||||
it('re-fetches the whole object when updatedProperties is null', () => {
|
||||
const plan = planMailboxFetches(page({ updated: ['old'], updatedProperties: null }));
|
||||
expect(plan.fullIds).toEqual(['old']);
|
||||
expect(plan.countOnlyIds).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('keyset progress', () => {
|
||||
it('requires STRICTLY greater, because `after` is spec-inclusive', () => {
|
||||
// RFC 8621 s4.4.1: receivedAt "must be the same or after this date-time to
|
||||
// match". So every page re-returns the boundary message, and equality is NOT
|
||||
// progress - treating it as progress would loop on that millisecond forever.
|
||||
expect(madeForwardProgress('2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z')).toBe(false);
|
||||
expect(madeForwardProgress('2026-01-01T00:00:00.001Z', '2026-01-01T00:00:00.000Z')).toBe(true);
|
||||
expect(madeForwardProgress(null, '2026-01-01T00:00:00.000Z')).toBe(false);
|
||||
expect(madeForwardProgress('2026-01-01T00:00:00.000Z', null)).toBe(true);
|
||||
});
|
||||
|
||||
it('advances exactly one millisecond in the last-resort rung', () => {
|
||||
expect(advanceOneMs('2026-01-01T00:00:00.000Z')).toBe('2026-01-01T00:00:00.001Z');
|
||||
// A malformed value must not become NaN and poison the cursor.
|
||||
expect(advanceOneMs('not-a-date')).toBe('not-a-date');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
backoffDelayMs, classify, escalationApplies, movesCursor, nextRung, rungValue,
|
||||
type ErrorClass,
|
||||
} from '../errors';
|
||||
|
||||
const ALL: ErrorClass[] = [
|
||||
'Transport', 'RateLimit', 'ServerTransient', 'RequestLimit', 'Auth', 'Fatal', 'StateInvalid',
|
||||
];
|
||||
|
||||
describe('exactly one class moves a cursor', () => {
|
||||
it('is StateInvalid, and nothing else', () => {
|
||||
// This is the single load-bearing property of the taxonomy. Every other class
|
||||
// leaves the cursor exactly where it was, which is what makes "a failure never
|
||||
// causes silent data loss" structural rather than aspirational.
|
||||
expect(ALL.filter(movesCursor)).toEqual(['StateInvalid']);
|
||||
});
|
||||
|
||||
it('escalates to a rebuild only for size/availability problems', () => {
|
||||
// Escalating on RateLimit would answer a rate-limited server with far MORE
|
||||
// requests. On Auth, a 401 would trigger a rebuild. On Transport, a flaky
|
||||
// tunnel would. Fatal is our own bug and a rebuild will not fix it.
|
||||
expect(ALL.filter(escalationApplies).sort()).toEqual(['RequestLimit', 'ServerTransient']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('classify', () => {
|
||||
it('reads HTTP status before anything else', () => {
|
||||
expect(classify({ httpStatus: 401 })).toBe('Auth');
|
||||
expect(classify({ httpStatus: 403 })).toBe('Auth');
|
||||
expect(classify({ httpStatus: 429 })).toBe('RateLimit');
|
||||
expect(classify({ httpStatus: 413 })).toBe('RequestLimit');
|
||||
expect(classify({ httpStatus: 503 })).toBe('ServerTransient');
|
||||
});
|
||||
|
||||
it('classifies cannotCalculateChanges as the one cursor-moving class', () => {
|
||||
expect(classify({ jmapErrorType: 'cannotCalculateChanges' })).toBe('StateInvalid');
|
||||
});
|
||||
|
||||
it('defaults an UNRECOGNISED method error to ServerTransient', () => {
|
||||
// Guessing transient costs a retry; guessing state-invalid costs a full
|
||||
// resync; guessing fatal stalls the account. The cheapest wrong answer wins.
|
||||
expect(classify({ jmapErrorType: 'somethingNobodyHasHeardOf' })).toBe('ServerTransient');
|
||||
});
|
||||
|
||||
it('does not let a method error description masquerade as a transport failure', () => {
|
||||
// Structure before strings: a method error's prose can legitimately contain
|
||||
// "timeout" or "socket", and reading that as Transport would leave a genuine
|
||||
// server-side problem being retried as though the network were down.
|
||||
expect(classify({ jmapErrorType: 'invalidArguments', message: 'socket timeout' })).toBe('Fatal');
|
||||
});
|
||||
|
||||
it('classifies a real fetch rejection as Transport', () => {
|
||||
// "Offline is not an error": the cursor stands still and the work is retried.
|
||||
for (const message of [
|
||||
'fetch failed', 'connect ECONNREFUSED 127.0.0.1:1', 'getaddrinfo ENOTFOUND nope',
|
||||
'socket hang up', 'The operation timed out',
|
||||
]) {
|
||||
expect(classify({ message }), message).toBe('Transport');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('the maxChanges ladder is monotonically non-increasing for EVERY server value', () => {
|
||||
it('never proposes a retry larger than the attempt that just failed', () => {
|
||||
// Two historical bugs live here. An unbounded middle rung produced a retry
|
||||
// STRICTLY LARGER than the failing attempt, actively worsening a
|
||||
// "response too large" error. Clamping only rung 0 then reintroduced it in a
|
||||
// narrower form: maxObjectsInGet=100 gave rung0=100 and rung1=250.
|
||||
const serverValues = [
|
||||
undefined, 1, 5, 10, 20, 25, 26, 49, 50, 51, 99, 100, 249, 250, 251, 499, 500, 501, 5000,
|
||||
];
|
||||
for (const value of serverValues) {
|
||||
const rungs = ([0, 1, 2, 3] as const).map((r) => rungValue(r, value));
|
||||
for (let i = 1; i < rungs.length; i++) {
|
||||
expect(
|
||||
rungs[i],
|
||||
`maxObjectsInGet=${value} rung ${i} (${rungs[i]}) must not exceed rung ${i - 1} (${rungs[i - 1]})`,
|
||||
).toBeLessThanOrEqual(rungs[i - 1]);
|
||||
}
|
||||
// And never zero, or the request asks for nothing and never progresses.
|
||||
for (const r of rungs) expect(r).toBeGreaterThanOrEqual(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('clamps rung 0 to what the server allows', () => {
|
||||
expect(rungValue(0, 100)).toBe(100);
|
||||
expect(rungValue(0, 5000)).toBe(500);
|
||||
expect(rungValue(0, undefined)).toBe(500);
|
||||
});
|
||||
|
||||
it('saturates rather than running off the end of the ladder', () => {
|
||||
expect(nextRung(0)).toBe(1);
|
||||
expect(nextRung(3)).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('backoff', () => {
|
||||
it('is full-jitter and bounded by the cap', () => {
|
||||
for (let attempt = 0; attempt < 12; attempt++) {
|
||||
const delay = backoffDelayMs(attempt, { baseMs: 1000, capMs: 60_000 });
|
||||
expect(delay).toBeGreaterThanOrEqual(0);
|
||||
expect(delay).toBeLessThanOrEqual(60_000);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
// The clock-jump guard, and the wipe it caused on the mobile client.
|
||||
//
|
||||
// The bug being regressed here is not hypothetical: its reproduction on the mobile
|
||||
// side returned 0 envelopes from a 4-envelope store. The guard DETECTED the jump,
|
||||
// held the old floor for one cycle - and persisted the JUMPED floor. The next
|
||||
// chained cycle seconds later computed a floor within seconds of the persisted
|
||||
// one, so the guard passed, the movement was classified as a NARROW, and every
|
||||
// envelope below a floor a year in the future was evicted. Unrecoverable, because
|
||||
// `coveredFrom` then claims the range complete and `/changes` cannot re-deliver
|
||||
// pre-existing mail.
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
adjustForWindow, CLOCK_JUMP_GUARD_MS, computeFloors, floorMovement,
|
||||
guardFloorAgainstClockJump,
|
||||
} from '../retention';
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
const T0 = Date.parse('2026-08-05T12:00:00.000Z');
|
||||
|
||||
function iso(t: number): string {
|
||||
return new Date(t).toISOString();
|
||||
}
|
||||
|
||||
describe('computeFloors', () => {
|
||||
it('never lets the body window be wider than the envelope window', () => {
|
||||
// A body with no envelope is an orphan by construction, and the whole point of
|
||||
// two tiers is envelopes being a superset of bodies.
|
||||
const floors = computeFloors({ envelopeDays: 30, bodyDays: 365, maxBodyMB: 100 }, T0);
|
||||
expect(floors.bodyFrom).toBe(floors.envelopeFrom);
|
||||
});
|
||||
|
||||
it('turns the MB cap into bytes', () => {
|
||||
expect(computeFloors({ envelopeDays: 1, bodyDays: 1, maxBodyMB: 2 }, T0).maxBodyBytes)
|
||||
.toBe(2 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe('guardFloorAgainstClockJump', () => {
|
||||
it('adopts the computed floor when there is no history to compare against', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0), undefined);
|
||||
expect(g.suppressed).toBe(false);
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
expect(g.envelopeFrom).toBe(iso(T0));
|
||||
});
|
||||
|
||||
it('adopts an ordinary drift - a DST shift must not trip it', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0 + 60 * 60 * 1000), iso(T0));
|
||||
expect(g.suppressed).toBe(false);
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
});
|
||||
|
||||
it('suppresses a jump larger than the guard and refuses to authorise deletion', () => {
|
||||
const jumped = iso(T0 + 365 * DAY);
|
||||
const g = guardFloorAgainstClockJump(jumped, iso(T0));
|
||||
expect(g.suppressed).toBe(true);
|
||||
expect(g.envelopeFrom).toBe(iso(T0));
|
||||
// Suppressing the FLOOR is not the same as suppressing the DELETIONS the
|
||||
// floor authorises. Both the retention eviction and the reconcile sweep read
|
||||
// this bit.
|
||||
expect(g.evictionAllowed).toBe(false);
|
||||
expect(g.warning).toBeTruthy();
|
||||
});
|
||||
|
||||
it('THE H2 REGRESSION: persists the floor it USED, not the one it rejected', () => {
|
||||
// This one assertion is the whole fix. Persisting the computed value here is
|
||||
// what legitimised the anomaly on the very next cycle.
|
||||
const jumped = iso(T0 + 365 * DAY);
|
||||
const g = guardFloorAgainstClockJump(jumped, iso(T0));
|
||||
expect(g.nextLastWindowFloor).toBe(iso(T0));
|
||||
expect(g.nextLastWindowFloor).not.toBe(jumped);
|
||||
});
|
||||
|
||||
it('THE H2 REGRESSION: stays suppressed across MANY chained cycles', () => {
|
||||
// The original bug only showed on the SECOND cycle, so a single-cycle test
|
||||
// passes against the broken code. Chaining is what reproduces it.
|
||||
const stored = iso(T0);
|
||||
let lastWindowFloor: string | undefined = stored;
|
||||
for (let cycle = 0; cycle < 20; cycle++) {
|
||||
// The clock is a year ahead and creeping forward a few seconds per cycle,
|
||||
// exactly as a chained sync would observe it.
|
||||
const computed = iso(T0 + 365 * DAY + cycle * 5_000);
|
||||
const g = guardFloorAgainstClockJump(computed, lastWindowFloor);
|
||||
expect(g.suppressed, `cycle ${cycle} must stay suppressed`).toBe(true);
|
||||
expect(g.evictionAllowed, `cycle ${cycle} must not authorise deletion`).toBe(false);
|
||||
expect(g.envelopeFrom, `cycle ${cycle} must keep the original floor`).toBe(stored);
|
||||
lastWindowFloor = g.nextLastWindowFloor;
|
||||
}
|
||||
// And after 20 cycles the remembered floor is still the trustworthy one, so
|
||||
// no later cycle can classify it as a narrow and evict everything.
|
||||
expect(lastWindowFloor).toBe(stored);
|
||||
expect(floorMovement(lastWindowFloor, stored)).toBe('unchanged');
|
||||
expect(adjustForWindow(floorMovement(lastWindowFloor, stored), stored).evictBelow)
|
||||
.toBeUndefined();
|
||||
});
|
||||
|
||||
it('suppresses a BACKWARD jump too', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0 - 365 * DAY), iso(T0));
|
||||
expect(g.suppressed).toBe(true);
|
||||
expect(g.evictionAllowed).toBe(false);
|
||||
});
|
||||
|
||||
it('treats an explicit retention change as INTENT and applies it, eviction included', () => {
|
||||
// The computed floor moves for two independent reasons - the clock changing
|
||||
// and the SETTING changing - and guarding a setting change is wrong. Without
|
||||
// this discriminator a Settings edit sits unapplied until something unrelated
|
||||
// moves the floor again.
|
||||
const widened = iso(T0 - 365 * DAY);
|
||||
const g = guardFloorAgainstClockJump(widened, iso(T0), { policyChanged: true });
|
||||
expect(g.suppressed).toBe(false);
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
expect(g.envelopeFrom).toBe(widened);
|
||||
expect(g.nextLastWindowFloor).toBe(widened);
|
||||
});
|
||||
|
||||
it('a genuine user NARROW still evicts', () => {
|
||||
// The guard must not become a reason nothing is ever deleted.
|
||||
const narrowed = iso(T0 + 20 * 60 * 60 * 1000);
|
||||
const g = guardFloorAgainstClockJump(narrowed, iso(T0));
|
||||
expect(g.evictionAllowed).toBe(true);
|
||||
expect(floorMovement(iso(T0), g.envelopeFrom)).toBe('narrowed');
|
||||
expect(adjustForWindow('narrowed', g.envelopeFrom).evictBelow).toBe(narrowed);
|
||||
});
|
||||
|
||||
it('tolerates an unparseable stored floor without wedging', () => {
|
||||
const g = guardFloorAgainstClockJump(iso(T0), 'garbage');
|
||||
// Date.parse('garbage') is NaN, so the delta is not finite: adopt rather than
|
||||
// suppress forever on a corrupt value.
|
||||
expect(g.suppressed).toBe(false);
|
||||
});
|
||||
|
||||
it('uses a threshold above a day so a leap second or NTP nudge is invisible', () => {
|
||||
expect(CLOCK_JUMP_GUARD_MS).toBeGreaterThan(DAY);
|
||||
});
|
||||
});
|
||||
|
||||
describe('floorMovement / adjustForWindow', () => {
|
||||
it('a LATER floor keeps less mail and means evict', () => {
|
||||
expect(floorMovement(iso(T0), iso(T0 + DAY))).toBe('narrowed');
|
||||
expect(adjustForWindow('narrowed', iso(T0 + DAY))).toEqual({ evictBelow: iso(T0 + DAY) });
|
||||
});
|
||||
|
||||
it('an EARLIER floor means re-scan, NOT a resync', () => {
|
||||
// A widen moves the target back and re-enters coverage scanning. The cursors
|
||||
// are untouched - a widen is not a reason to rebuild.
|
||||
expect(floorMovement(iso(T0), iso(T0 - DAY))).toBe('widened');
|
||||
expect(adjustForWindow('widened', iso(T0 - DAY))).toEqual({ rescanFrom: iso(T0 - DAY) });
|
||||
});
|
||||
|
||||
it('does nothing without a previous floor', () => {
|
||||
expect(floorMovement(undefined, iso(T0))).toBe('unchanged');
|
||||
expect(adjustForWindow('unchanged', iso(T0))).toEqual({});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
asChangesState, asSnapshotState, coveragePhaseForCommitment, mintEnumerationCommitment,
|
||||
} from '../states';
|
||||
|
||||
describe('state token certification', () => {
|
||||
it('rejects everything a parsed JSON body could hand over that is not a token', () => {
|
||||
// The brand certifies PROVENANCE; this check certifies SHAPE. Without it a
|
||||
// `null` or a number could be laundered into something the engine treats as a
|
||||
// cursor forever.
|
||||
for (const bad of [null, undefined, 0, 1, '', {}, [], true]) {
|
||||
expect(() => asChangesState(bad)).toThrow(TypeError);
|
||||
expect(() => asSnapshotState(bad)).toThrow(TypeError);
|
||||
}
|
||||
});
|
||||
|
||||
it('accepts a non-empty string', () => {
|
||||
expect(asChangesState('s1')).toBe('s1');
|
||||
expect(asSnapshotState('s1')).toBe('s1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('EnumerationCommitment', () => {
|
||||
it('is constructible - the symbol tag must be a real runtime Symbol', () => {
|
||||
// `declare const tag: unique symbol` is type-level only and emits no runtime
|
||||
// value, so using it as a computed key throws ReferenceError the first time
|
||||
// the mint runs. That mistake is in the superseded design document; this test
|
||||
// is what catches it.
|
||||
const commitment = mintEnumerationCommitment({
|
||||
jmapAccountId: 'a',
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: '2026-01-01T00:00:00.000Z',
|
||||
sweepFloor: '2026-01-01T00:00:00.000Z',
|
||||
kind: 'bootstrap',
|
||||
});
|
||||
expect(commitment.snapshot).toBe('snap');
|
||||
expect(commitment.kind).toBe('bootstrap');
|
||||
});
|
||||
|
||||
it('maps its kind onto the coverage phase', () => {
|
||||
const base = {
|
||||
jmapAccountId: 'a',
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: 'x',
|
||||
sweepFloor: 'x',
|
||||
} as const;
|
||||
expect(coveragePhaseForCommitment(mintEnumerationCommitment({ ...base, kind: 'bootstrap' })))
|
||||
.toBe('scanning');
|
||||
expect(coveragePhaseForCommitment(mintEnumerationCommitment({ ...base, kind: 'reconcile' })))
|
||||
.toBe('reconciling');
|
||||
});
|
||||
|
||||
it('does not export its tag, so no object literal elsewhere can forge the type', () => {
|
||||
const source = fs.readFileSync(path.join(__dirname, '..', 'states.ts'), 'utf8');
|
||||
expect(source).toContain("const enumerationCommitmentTag = Symbol('EnumerationCommitment')");
|
||||
expect(source).not.toMatch(/export\s+(const|let)\s+enumerationCommitmentTag/);
|
||||
// And it must be a real Symbol() call, not the type-only declaration form.
|
||||
expect(source).not.toMatch(/declare\s+const\s+enumerationCommitmentTag/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cursor provenance is greppable, not just documented', () => {
|
||||
const replicaDir = path.join(__dirname, '..');
|
||||
|
||||
function sourceFiles(): string[] {
|
||||
return fs
|
||||
.readdirSync(replicaDir)
|
||||
.filter((f) => f.endsWith('.ts'))
|
||||
.map((f) => path.join(replicaDir, f));
|
||||
}
|
||||
|
||||
it('mints branded states ONLY in jmap.ts (the response parser)', () => {
|
||||
// This is the rule the whole brand exists to enforce. The mobile client's
|
||||
// defect D4 was a snapshot state adopted as a /changes cursor after a
|
||||
// transient 503; a cast anywhere outside the parser is how that comes back.
|
||||
for (const file of sourceFiles()) {
|
||||
const base = path.basename(file);
|
||||
if (base === 'states.ts' || base === 'jmap.ts') continue;
|
||||
const source = fs.readFileSync(file, 'utf8');
|
||||
expect(source, `${base} must not mint a ChangesState`).not.toMatch(/asChangesState\s*\(/);
|
||||
expect(source, `${base} must not mint a SnapshotState`).not.toMatch(/asSnapshotState\s*\(/);
|
||||
expect(source, `${base} must not cast to a branded state`).not.toMatch(
|
||||
/as\s+(ChangesState|SnapshotState)\b/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('mints an EnumerationCommitment ONLY where an enumeration is actually started', () => {
|
||||
// A commitment is a promise to enumerate. Minting one anywhere that does not
|
||||
// then enumerate makes the seed path's teeth meaningless.
|
||||
const callers = sourceFiles().filter((file) => {
|
||||
if (path.basename(file) === 'states.ts') return false;
|
||||
return /mintEnumerationCommitment\s*\(/.test(fs.readFileSync(file, 'utf8'));
|
||||
});
|
||||
expect(callers.map((f) => path.basename(f))).toEqual(['sync.ts']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,502 @@
|
||||
// Store-level invariants, against a REAL SQLCipher file.
|
||||
//
|
||||
// Skipped wholesale when the optional native binding is not installed (that is a
|
||||
// normal state on a platform with no prebuild - see lib/mail-index/binding.ts), so
|
||||
// this file must never be the only proof of anything.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
||||
import { indexDbPath } from '@/lib/mail-index/paths';
|
||||
import { clampPolicy, DEFAULT_POLICY, ReplicaStore } from '../store';
|
||||
import { reconcileStamp } from '../sync';
|
||||
import { asChangesState, asSnapshotState, mintEnumerationCommitment } from '../states';
|
||||
import type { EnvelopeRow } from '../types';
|
||||
|
||||
const ACCOUNT = 'alice@example.org';
|
||||
const JMAP = 'jmap-account-1';
|
||||
|
||||
function envelope(id: string, receivedAt: string, extra: Partial<EnvelopeRow> = {}): EnvelopeRow {
|
||||
return {
|
||||
jmapAccountId: JMAP,
|
||||
id,
|
||||
threadId: `t-${id}`,
|
||||
receivedAt,
|
||||
size: 1000,
|
||||
subject: `subject ${id}`,
|
||||
preview: `preview ${id}`,
|
||||
fromJson: JSON.stringify([{ email: 'sender@example.org' }]),
|
||||
toJson: null,
|
||||
ccJson: null,
|
||||
blobId: `blob-${id}`,
|
||||
hasAttachment: false,
|
||||
keywordsJson: '{}',
|
||||
mailboxIds: ['inbox'],
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
describe.skipIf(!isSqlcipherAvailable())('ReplicaStore', () => {
|
||||
let storeDir: string;
|
||||
let key: Buffer;
|
||||
let store: ReplicaStore;
|
||||
|
||||
beforeEach(() => {
|
||||
storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-test-'));
|
||||
key = randomBytes(32);
|
||||
store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
store.close();
|
||||
fs.rmSync(storeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes into the SAME file as the search index, and it is really encrypted', () => {
|
||||
// One encryption boundary, one key, one purge. And `PRAGMA key` is a silent
|
||||
// no-op on a non-SQLCipher binding, so the header check is the only thing that
|
||||
// catches a store that "works" while sitting on disk in cleartext.
|
||||
expect(store.dbPath).toBe(indexDbPath(storeDir, ACCOUNT));
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.close();
|
||||
const header = fs.readFileSync(store.dbPath).subarray(0, 15).toString('latin1');
|
||||
expect(header).not.toBe('SQLite format 3');
|
||||
const raw = Buffer.concat(
|
||||
['', '-wal', '-shm']
|
||||
.map((s) => `${store.dbPath}${s}`)
|
||||
.filter((f) => fs.existsSync(f))
|
||||
.map((f) => fs.readFileSync(f)),
|
||||
);
|
||||
expect(raw.includes('subject e1')).toBe(false);
|
||||
// Re-open so afterEach's close() is harmless.
|
||||
store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
|
||||
});
|
||||
|
||||
describe('cursor provenance at the storage layer', () => {
|
||||
it('refuses to create a cursor from nowhere', () => {
|
||||
// A cursor is born from seedCursor and nowhere else. Creating one in
|
||||
// advanceCursor would be a silent cursor-from-nowhere - exactly what the
|
||||
// branded types exist to make impossible.
|
||||
expect(() => store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s1')))
|
||||
.toThrow(/seed it first/);
|
||||
});
|
||||
|
||||
it('writes the cursor AND the coverage row it justifies in one transaction', () => {
|
||||
store.transaction(() => {
|
||||
store.seedCursor(
|
||||
{ jmapAccountId: JMAP, type: 'Email' },
|
||||
mintEnumerationCommitment({
|
||||
jmapAccountId: JMAP,
|
||||
snapshot: asSnapshotState('snap-1'),
|
||||
targetFrom: '2026-01-01T00:00:00.000Z',
|
||||
sweepFloor: '2026-01-01T00:00:00.000Z',
|
||||
kind: 'bootstrap',
|
||||
}),
|
||||
1000,
|
||||
);
|
||||
});
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('snap-1');
|
||||
const coverage = store.getCoverage(JMAP);
|
||||
expect(coverage?.phase).toBe('scanning');
|
||||
expect(coverage?.sweepFloor).toBe('2026-01-01T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('rolls back a seed whose commitment is for the wrong account', () => {
|
||||
expect(() =>
|
||||
store.transaction(() => {
|
||||
store.seedCursor(
|
||||
{ jmapAccountId: JMAP, type: 'Email' },
|
||||
mintEnumerationCommitment({
|
||||
jmapAccountId: 'someone-else',
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: 'x', sweepFloor: 'x', kind: 'bootstrap',
|
||||
}),
|
||||
1000,
|
||||
);
|
||||
}),
|
||||
).toThrow(/different JMAP account/);
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull();
|
||||
});
|
||||
|
||||
it('advances a seeded cursor and keeps counters field-level', () => {
|
||||
seed(store);
|
||||
store.transaction(() => {
|
||||
store.advanceCursor({ jmapAccountId: JMAP, type: 'Email' }, asChangesState('s2'));
|
||||
store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { consecutiveFailures: 3 });
|
||||
});
|
||||
const cursor = store.getCursor({ jmapAccountId: JMAP, type: 'Email' });
|
||||
expect(cursor?.state).toBe('s2');
|
||||
expect(cursor?.consecutiveFailures).toBe(3);
|
||||
// A patch must not be able to rewrite `state` - only advance/seed can.
|
||||
store.transaction(() => {
|
||||
store.patchCursor({ jmapAccountId: JMAP, type: 'Email' }, { drainPending: true });
|
||||
});
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })?.state).toBe('s2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('envelope tier', () => {
|
||||
it('does NOT reset has_body on an idempotent replay', () => {
|
||||
// Otherwise a replayed page looks like "body missing" to the backfill job and
|
||||
// re-downloads every body in the page.
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{}}'); });
|
||||
expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10)).toHaveLength(0);
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 2); });
|
||||
expect(
|
||||
store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10),
|
||||
'a replayed envelope upsert must not clear has_body',
|
||||
).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('patches only the two mutable properties, and no-ops for an absent id', () => {
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
const ok = store.transaction(() =>
|
||||
store.patchEnvelopeMutable(JMAP, 'e1', { keywordsJson: '{"$seen":true}', mailboxIds: ['archive'] }),
|
||||
);
|
||||
expect(ok).toBe(true);
|
||||
expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual(['archive']);
|
||||
// An update for an id we do not hold must leave no membership rows behind.
|
||||
const missing = store.transaction(() =>
|
||||
store.patchEnvelopeMutable(JMAP, 'nope', { keywordsJson: '{}', mailboxIds: ['inbox'] }),
|
||||
);
|
||||
expect(missing).toBe(false);
|
||||
expect(store.mailboxIdsFor(JMAP, 'nope')).toEqual([]);
|
||||
});
|
||||
|
||||
it('never writes a body whose envelope is gone', () => {
|
||||
// A body fetched moments before its envelope was destroyed in the same cycle
|
||||
// would otherwise land as an orphan.
|
||||
const wrote = store.transaction(() => store.putBodyIfEnvelopeExists(JMAP, 'ghost', '{}'));
|
||||
expect(wrote).toBe(false);
|
||||
expect(store.getBody(JMAP, 'ghost')).toBeNull();
|
||||
});
|
||||
|
||||
it('deleting an email takes its body, membership and queue row with it', () => {
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1);
|
||||
store.enqueueBodies([{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 }]);
|
||||
});
|
||||
store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"a":1}'); });
|
||||
store.transaction(() => { store.deleteEmails(JMAP, ['e1']); });
|
||||
expect(store.getBody(JMAP, 'e1')).toBeNull();
|
||||
expect(store.mailboxIdsFor(JMAP, 'e1')).toEqual([]);
|
||||
expect(store.countWantedBodies(JMAP, Date.now())).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the reconcile sweep', () => {
|
||||
it('refuses to run without a pinned stamp rather than deleting unverified records', () => {
|
||||
expect(() => store.sweep(JMAP, '2026-01-01T00:00:00.000Z', undefined))
|
||||
.toThrow(/refusing to delete unverified/);
|
||||
});
|
||||
|
||||
it('keeps what the enumeration re-saw and deletes what it did not', () => {
|
||||
// The whole "seen set as one integer" trick: re-upserting refreshes
|
||||
// cached_at, and the sweep deletes anything still below the pin.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('kept', '2026-08-01T00:00:00.000Z'),
|
||||
envelope('gone', '2026-08-02T00:00:00.000Z'),
|
||||
], 100);
|
||||
});
|
||||
const stamp = Math.max(500, store.maxEnvelopeCachedAt(JMAP) + 1);
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp);
|
||||
});
|
||||
store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); });
|
||||
expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull();
|
||||
expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull();
|
||||
});
|
||||
|
||||
it('a stamp taken from a FROZEN clock would sweep nothing; the derived one works', () => {
|
||||
// Both halves matter, and both fail silently. Exercising the real
|
||||
// `reconcileStamp` rather than re-deriving it in the test is the point.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('kept', '2026-08-01T00:00:00.000Z'),
|
||||
envelope('gone', '2026-08-02T00:00:00.000Z'),
|
||||
], 9_999);
|
||||
});
|
||||
const frozenNow = 1_000;
|
||||
|
||||
// The naive version: with the clock behind the data, nothing is below the
|
||||
// stamp, so a re-verified store sweeps zero rows and stale records live on.
|
||||
expect(store.sweep(JMAP, '2026-07-01T00:00:00.000Z', frozenNow)).toBe(0);
|
||||
|
||||
const stamp = reconcileStamp(frozenNow, store.maxEnvelopeCachedAt(JMAP));
|
||||
expect(stamp).toBe(10_000);
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('kept', '2026-08-01T00:00:00.000Z')], stamp);
|
||||
});
|
||||
expect(store.transaction(() => store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp))).toBe(1);
|
||||
expect(store.getEnvelopeRaw(JMAP, 'kept')).not.toBeNull();
|
||||
expect(store.getEnvelopeRaw(JMAP, 'gone')).toBeNull();
|
||||
});
|
||||
|
||||
it('stamping an enumeration with `now` instead of the pin deletes what it just verified', () => {
|
||||
// The other direction of the same bug: the pin EXCEEDS now, so a page that
|
||||
// stamps with `now` lands below the pin and the sweep eats it.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], 9_999);
|
||||
});
|
||||
const now = 1_000;
|
||||
const stamp = reconcileStamp(now, store.maxEnvelopeCachedAt(JMAP));
|
||||
// Re-verified against the server, but stamped with the WRONG value.
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([envelope('verified', '2026-08-01T00:00:00.000Z')], now);
|
||||
});
|
||||
store.transaction(() => { store.sweep(JMAP, '2026-07-01T00:00:00.000Z', stamp); });
|
||||
expect(
|
||||
store.getEnvelopeRaw(JMAP, 'verified'),
|
||||
'this is the failure mode the pinned stamp exists to prevent',
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the body queue - the durable-terminal-state fixes', () => {
|
||||
it('enqueueBodies reports rows ACTUALLY INSERTED, not attempted', () => {
|
||||
// Reporting the attempted count made the mobile engine believe there was
|
||||
// unfinished work every cycle for as long as any envelope lacked a body,
|
||||
// chaining a new cycle every few seconds indefinitely.
|
||||
const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 };
|
||||
expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(1);
|
||||
expect(store.transaction(() => store.enqueueBodies([entry]))).toBe(0);
|
||||
});
|
||||
|
||||
it('never resets attempts on a re-enqueue', () => {
|
||||
const entry = { emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 };
|
||||
store.transaction(() => { store.enqueueBodies([entry]); });
|
||||
store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 0, 'boom'); });
|
||||
store.transaction(() => { store.enqueueBodies([entry]); });
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())[0]?.attempts).toBe(1);
|
||||
});
|
||||
|
||||
it('THE H1 REGRESSION: a gave-up row is KEPT and is never revived by a re-enqueue', () => {
|
||||
// Deleting the row on give-up was not enough: the backfill driver is
|
||||
// "envelope with no body", which cannot tell "not fetched yet" from
|
||||
// "deliberately not kept", so the next pass re-inserted a fresh attempts=0
|
||||
// row and a permanently-failing body was retried five times per cycle forever.
|
||||
store.transaction(() => {
|
||||
store.markBodyGaveUp(JMAP, [
|
||||
{ emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' },
|
||||
]);
|
||||
});
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['e1']);
|
||||
// Not WANTED any more, so the drain never picks it up again.
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0);
|
||||
// And a re-enqueue cannot resurrect it.
|
||||
const inserted = store.transaction(() =>
|
||||
store.enqueueBodies([
|
||||
{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
|
||||
]),
|
||||
);
|
||||
expect(inserted).toBe(0);
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('THE H1c REGRESSION: a cap-shed body is markable even with NO existing queue row', () => {
|
||||
// The download/discard loop: the cap sheds a body that was fetched and stored
|
||||
// successfully, so there is no queue row left to UPDATE. If the mark is
|
||||
// silently dropped, the envelope is still inside the body WINDOW, the backfill
|
||||
// re-enqueues it, it downloads again, and the cap sheds it again - unbounded
|
||||
// data use that never terminates.
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.transaction(() => { store.putBodyIfEnvelopeExists(JMAP, 'e1', '{"bodyValues":{"1":{"value":"x"}}}'); });
|
||||
expect(store.takeBodyQueue(JMAP, 10, Date.now())).toHaveLength(0); // no queue row exists
|
||||
|
||||
store.transaction(() => {
|
||||
store.deleteBodies(JMAP, ['e1']);
|
||||
store.markBodyGaveUp(JMAP, [
|
||||
{ emailId: 'e1', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' },
|
||||
]);
|
||||
});
|
||||
expect(
|
||||
store.listBodyGiveUps(JMAP, 10),
|
||||
'the cap-shed mark must be an upsert, or the shed/re-download loop stays open',
|
||||
).toEqual(['e1']);
|
||||
// The envelope is back to has_body=0 and still in the window, so without the
|
||||
// mark the backfill WOULD pick it up. With the mark it is excluded.
|
||||
expect(store.envelopesWithoutBody(JMAP, '2026-01-01T00:00:00.000Z', 10).map((e) => e.id))
|
||||
.toEqual(['e1']);
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toContain('e1');
|
||||
});
|
||||
|
||||
it('clearing give-ups DELETES them, so they look like "never queued"', () => {
|
||||
// A cleared give-up must come back with a clean attempt count, which an
|
||||
// un-flag would not give.
|
||||
store.transaction(() => {
|
||||
store.markBodyGaveUp(JMAP, [
|
||||
{ emailId: 'a', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'attempts' },
|
||||
{ emailId: 'b', receivedAt: '2026-08-01T00:00:00.000Z', reason: 'shed-by-cap' },
|
||||
]);
|
||||
});
|
||||
store.transaction(() => { store.clearBodyGiveUps(JMAP, 'shed-by-cap'); });
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toEqual(['a']);
|
||||
store.transaction(() => { store.clearBodyGiveUps(JMAP); });
|
||||
expect(store.listBodyGiveUps(JMAP, 10)).toEqual([]);
|
||||
expect(
|
||||
store.transaction(() =>
|
||||
store.enqueueBodies([
|
||||
{ emailId: 'a', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
|
||||
]),
|
||||
),
|
||||
'a cleared give-up must be re-enqueueable',
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it('honours a backoff window', () => {
|
||||
store.transaction(() => {
|
||||
store.enqueueBodies([
|
||||
{ emailId: 'e1', jmapAccountId: JMAP, receivedAt: '2026-08-01T00:00:00.000Z', attempts: 0 },
|
||||
]);
|
||||
});
|
||||
store.transaction(() => { store.bumpBodyAttempt(JMAP, 'e1', 10_000, 'later'); });
|
||||
expect(store.takeBodyQueue(JMAP, 10, 5_000)).toHaveLength(0);
|
||||
expect(store.takeBodyQueue(JMAP, 10, 20_000)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('eviction', () => {
|
||||
it('cap eviction takes the oldest bodies first and leaves envelopes alone', () => {
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('old', '2026-01-01T00:00:00.000Z'),
|
||||
envelope('new', '2026-08-01T00:00:00.000Z'),
|
||||
], 1);
|
||||
});
|
||||
store.transaction(() => {
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'old', '{"v":"old"}');
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'new', '{"v":"new"}');
|
||||
});
|
||||
expect(store.oldestBodies(JMAP, 1).map((b) => b.emailId)).toEqual(['old']);
|
||||
store.transaction(() => { store.deleteBodies(JMAP, ['old']); });
|
||||
// The message stays LISTED - only its content went.
|
||||
expect(store.getEnvelopeRaw(JMAP, 'old')).not.toBeNull();
|
||||
expect(store.countBodies(JMAP)).toBe(1);
|
||||
});
|
||||
|
||||
it('no deletion path leaves an orphan body behind', () => {
|
||||
// This is the real invariant. `orphanBodies()` is a belt-and-braces sweep for
|
||||
// orphans a CRASH between two transactions could leave; it is deliberately
|
||||
// not reachable through the store's own API, which is what this asserts.
|
||||
// (So the detection query itself is covered only by the integration run, not
|
||||
// by this file - stated rather than papered over with a vacuous assertion.)
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('a', '2026-01-01T00:00:00.000Z'),
|
||||
envelope('b', '2026-08-01T00:00:00.000Z'),
|
||||
], 1);
|
||||
});
|
||||
store.transaction(() => {
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'a', '{"v":1}');
|
||||
store.putBodyIfEnvelopeExists(JMAP, 'b', '{"v":2}');
|
||||
});
|
||||
expect(store.countBodies(JMAP)).toBe(2);
|
||||
|
||||
store.transaction(() => { store.deleteEmails(JMAP, ['a']); });
|
||||
expect(store.orphanBodies(JMAP, 10)).toEqual([]);
|
||||
|
||||
store.transaction(() => { store.evictEnvelopesBelow(JMAP, '2026-09-01T00:00:00.000Z'); });
|
||||
expect(store.countEnvelopes(JMAP)).toBe(0);
|
||||
expect(store.countBodies(JMAP)).toBe(0);
|
||||
expect(store.orphanBodies(JMAP, 10)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('purge', () => {
|
||||
it('purgeAll takes the CURSORS with the records', () => {
|
||||
// A record wipe that leaves a live cursor behind is the one state no amount
|
||||
// of syncing repairs: /changes cannot re-deliver mail that already existed
|
||||
// when the cursor was captured.
|
||||
seed(store);
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.transaction(() => { store.purgeAll(); });
|
||||
expect(store.countEnvelopes(JMAP)).toBe(0);
|
||||
expect(store.getCursor({ jmapAccountId: JMAP, type: 'Email' })).toBeNull();
|
||||
expect(store.getCoverage(JMAP)).toBeNull();
|
||||
});
|
||||
|
||||
it('a wrong key is treated as unreadable and rebuilt, never as a prompt', () => {
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('e1', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
store.close();
|
||||
const other = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key: randomBytes(32) });
|
||||
try {
|
||||
expect(other.countEnvelopes(JMAP)).toBe(0);
|
||||
} finally {
|
||||
other.close();
|
||||
}
|
||||
store = ReplicaStore.open({ storeDir, accountId: ACCOUNT, key });
|
||||
});
|
||||
});
|
||||
|
||||
describe('policy', () => {
|
||||
it('round-trips and clamps', () => {
|
||||
store.transaction(() => { store.setPolicy({ envelopeDays: 99999, bodyDays: 0, maxBodyMB: 1 }); });
|
||||
const policy = store.getPolicy();
|
||||
expect(policy.envelopeDays).toBe(3650);
|
||||
expect(policy.bodyDays).toBe(1);
|
||||
expect(policy.maxBodyMB).toBe(16);
|
||||
});
|
||||
|
||||
it('defaults when nothing was ever written', () => {
|
||||
expect(store.getPolicy()).toEqual(DEFAULT_POLICY);
|
||||
});
|
||||
});
|
||||
|
||||
describe('read path', () => {
|
||||
it('lists a mailbox page newest-first with a correct total', () => {
|
||||
store.transaction(() => {
|
||||
store.upsertEnvelopes([
|
||||
envelope('a', '2026-08-01T00:00:00.000Z'),
|
||||
envelope('b', '2026-08-02T00:00:00.000Z'),
|
||||
envelope('c', '2026-08-03T00:00:00.000Z', { mailboxIds: ['archive'] }),
|
||||
], 1);
|
||||
});
|
||||
const inbox = store.listEnvelopes(JMAP, 'inbox', 10, 0);
|
||||
expect(inbox.total).toBe(2);
|
||||
expect(inbox.rows.map((r) => String(r.id))).toEqual(['b', 'a']);
|
||||
// A null mailbox is "everything", which is what the unified views want.
|
||||
expect(store.listEnvelopes(JMAP, null, 10, 0).total).toBe(3);
|
||||
expect(store.listEnvelopes(JMAP, 'archive', 10, 0).rows.map((r) => String(r.id))).toEqual(['c']);
|
||||
});
|
||||
|
||||
it('reports the account ids it holds without needing a network session', () => {
|
||||
store.transaction(() => { store.upsertEnvelopes([envelope('a', '2026-08-01T00:00:00.000Z')], 1); });
|
||||
expect(store.knownJmapAccountIds()).toEqual([JMAP]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('clampPolicy', () => {
|
||||
it('never lets the body window exceed the envelope window', () => {
|
||||
expect(clampPolicy({ envelopeDays: 30, bodyDays: 365, maxBodyMB: 100 }).bodyDays).toBe(30);
|
||||
});
|
||||
|
||||
it('falls back to defaults for junk input', () => {
|
||||
expect(clampPolicy({ envelopeDays: NaN } as never).envelopeDays).toBe(DEFAULT_POLICY.envelopeDays);
|
||||
expect(clampPolicy(null)).toEqual(DEFAULT_POLICY);
|
||||
expect(clampPolicy(undefined)).toEqual(DEFAULT_POLICY);
|
||||
});
|
||||
});
|
||||
|
||||
function seed(store: ReplicaStore): void {
|
||||
store.transaction(() => {
|
||||
for (const type of ['Email', 'Mailbox'] as const) {
|
||||
store.seedCursor(
|
||||
{ jmapAccountId: JMAP, type },
|
||||
mintEnumerationCommitment({
|
||||
jmapAccountId: JMAP,
|
||||
snapshot: asSnapshotState('snap'),
|
||||
targetFrom: '2026-01-01T00:00:00.000Z',
|
||||
sweepFloor: '2026-01-01T00:00:00.000Z',
|
||||
kind: 'bootstrap',
|
||||
}),
|
||||
1000,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// Change-application planning. PURE: no network, no storage, no store access.
|
||||
//
|
||||
// That purity is the single highest-leverage constraint in the design, because
|
||||
// `plan(page, presentIds) -> what to fetch` is assertable as plain data. It is
|
||||
// what turns a failure-mode table into a test suite rather than a promise.
|
||||
|
||||
import type { ChangesState } from './states';
|
||||
|
||||
export interface ChangesPage {
|
||||
oldState: ChangesState;
|
||||
newState: ChangesState;
|
||||
hasMoreChanges: boolean;
|
||||
created: string[];
|
||||
updated: string[];
|
||||
destroyed: string[];
|
||||
/** `Mailbox/changes` only. `null`/absent means "assume everything changed". */
|
||||
updatedProperties?: string[] | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapses the overlap RFC 8620 permits, BEFORE anything iterates - so
|
||||
* downstream code cannot get the order wrong by following the server's array
|
||||
* order.
|
||||
*/
|
||||
export function normalisePage(page: ChangesPage): {
|
||||
created: string[];
|
||||
updated: string[];
|
||||
destroyed: string[];
|
||||
} {
|
||||
const destroyed = [...new Set(page.destroyed)];
|
||||
const destroyedSet = new Set(destroyed);
|
||||
// An id in `destroyed` wins outright: fetching it would be wasted and the
|
||||
// result would be `notFound`.
|
||||
const created = [...new Set(page.created)].filter((id) => !destroyedSet.has(id));
|
||||
const createdSet = new Set(created);
|
||||
// An id in both `created` and `updated` is a CREATE - the create path fetches
|
||||
// the full envelope tier, which already includes the updated values.
|
||||
const updated = [...new Set(page.updated)].filter(
|
||||
(id) => !destroyedSet.has(id) && !createdSet.has(id),
|
||||
);
|
||||
return { created, updated, destroyed };
|
||||
}
|
||||
|
||||
/**
|
||||
* An empty page STILL ADVANCES THE CURSOR. Skipping it re-requests the same
|
||||
* position forever.
|
||||
*/
|
||||
export function pageIsEmpty(page: ChangesPage): boolean {
|
||||
return page.created.length === 0 && page.updated.length === 0 && page.destroyed.length === 0;
|
||||
}
|
||||
|
||||
export interface EmailFetchPlan {
|
||||
/** Full envelope tier. */
|
||||
createIds: string[];
|
||||
/** THREE properties only: id, keywords, mailboxIds. Never bodies. */
|
||||
updateIds: string[];
|
||||
destroyIds: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* `keywords` and `mailboxIds` are the ONLY mutable Email properties
|
||||
* (RFC 8621 s4.1). Body structure, body values, attachments, headers,
|
||||
* `receivedAt`, `size`, `threadId`, `preview`, `subject`, addresses and
|
||||
* `hasAttachment` are all immutable for the lifetime of the id.
|
||||
*
|
||||
* So an `updated` Email cannot have a changed body, and re-fetching one is pure
|
||||
* waste. This is also what stops a message cached while unread from staying
|
||||
* unread forever.
|
||||
*
|
||||
* An `updated` id we do NOT hold locally is an UNCONDITIONAL NO-OP, filtered out
|
||||
* BEFORE the fetch is issued. Cheaper, and it avoids having to fabricate a
|
||||
* `receivedAt` that a 3-property response cannot supply and the schema's NOT NULL
|
||||
* would reject. Safe to ignore because absence is always either "retention
|
||||
* decided against it" or "coverage has not reached it yet" - and coverage
|
||||
* enumerates CURRENT state, so it will pick the record up with the updated values
|
||||
* anyway. Nothing needs the update replayed.
|
||||
*/
|
||||
export function planEmailFetches(
|
||||
page: ChangesPage,
|
||||
presentIds: ReadonlySet<string>,
|
||||
): EmailFetchPlan {
|
||||
const { created, updated, destroyed } = normalisePage(page);
|
||||
return {
|
||||
createIds: created,
|
||||
updateIds: updated.filter((id) => presentIds.has(id)),
|
||||
destroyIds: destroyed,
|
||||
};
|
||||
}
|
||||
|
||||
const COUNT_PROPERTIES = new Set([
|
||||
'totalEmails', 'unreadEmails', 'totalThreads', 'unreadThreads',
|
||||
]);
|
||||
|
||||
/**
|
||||
* True when a `Mailbox/changes` update touched only the four counters, so a
|
||||
* four-integer patch is enough instead of re-fetching every folder object.
|
||||
*
|
||||
* `updatedProperties: null` means the server will not say, so everything must be
|
||||
* re-fetched. An EMPTY array means "nothing but the state token moved", which is
|
||||
* counts-only vacuously.
|
||||
*/
|
||||
export function updatedPropertiesAreCountsOnly(
|
||||
updatedProperties: readonly string[] | null | undefined,
|
||||
): boolean {
|
||||
if (!updatedProperties) return false;
|
||||
if (updatedProperties.length === 0) return true;
|
||||
return updatedProperties.every((p) => COUNT_PROPERTIES.has(p));
|
||||
}
|
||||
|
||||
export interface MailboxFetchPlan {
|
||||
/** Needs the whole object. */
|
||||
fullIds: string[];
|
||||
/** Only the count columns move. */
|
||||
countOnlyIds: string[];
|
||||
destroyIds: string[];
|
||||
}
|
||||
|
||||
export function planMailboxFetches(page: ChangesPage): MailboxFetchPlan {
|
||||
const { created, updated, destroyed } = normalisePage(page);
|
||||
const countsOnly = updatedPropertiesAreCountsOnly(page.updatedProperties);
|
||||
return {
|
||||
fullIds: countsOnly ? created : [...created, ...updated],
|
||||
countOnlyIds: countsOnly ? updated : [],
|
||||
destroyIds: destroyed,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyset-walk progress test.
|
||||
*
|
||||
* `after` is INCLUSIVE - this is specified, not implementation-defined.
|
||||
* RFC 8621 s4.4.1: the `receivedAt` of the Email "must be the same or after this
|
||||
* date-time to match the condition". So every page after the first re-returns the
|
||||
* boundary message(s); dedupe by id on commit makes that free. But forward
|
||||
* progress therefore requires `max(receivedAt)` STRICTLY GREATER than the cursor.
|
||||
*
|
||||
* Treating `after` as exclusive and adding a millisecond, as an earlier revision
|
||||
* of the mobile design did, silently skips every message sharing the boundary
|
||||
* millisecond on any conforming server.
|
||||
*/
|
||||
export function madeForwardProgress(
|
||||
maxReceivedAt: string | null,
|
||||
scanCursor: string | null,
|
||||
): boolean {
|
||||
if (maxReceivedAt === null) return false;
|
||||
if (scanCursor === null) return true;
|
||||
return maxReceivedAt > scanCursor;
|
||||
}
|
||||
|
||||
/** Advance a scan cursor by exactly one millisecond. The last-resort paging rung. */
|
||||
export function advanceOneMs(iso: string): string {
|
||||
const t = Date.parse(iso);
|
||||
if (!Number.isFinite(t)) return iso;
|
||||
return new Date(t + 1).toISOString();
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
// Session resolution, store lifecycle and single-flight. The thin layer every
|
||||
// `/api/offline/*` replica route goes through.
|
||||
|
||||
import { logger } from '@/lib/logger';
|
||||
import { fetchJmapSession, accountIdFor, CAP_MAIL, type JmapSessionInfo } from '@/lib/mail-index/jmap';
|
||||
import { withIndexKey } from '@/lib/mail-index/key';
|
||||
import { getStoreDir } from '@/lib/mail-index/paths';
|
||||
import {
|
||||
IndexSessionError, resolveIndexSession, type IndexSession,
|
||||
} from '@/lib/mail-index/reindex';
|
||||
import { classify, ReplicaSyncError } from './errors';
|
||||
import { ReplicaStore, type RetentionPolicy } from './store';
|
||||
import { BUDGET, runCycle, type CycleReport } from './sync';
|
||||
|
||||
export { IndexSessionError, resolveIndexSession };
|
||||
export type { IndexSession };
|
||||
|
||||
/**
|
||||
* Opens the replica for one operation and closes it afterwards.
|
||||
*
|
||||
* The key is fetched from the main process over the inherited fd for the duration
|
||||
* of the call only and zeroed after (`withIndexKey`) - there is no cached handle
|
||||
* and no resident key. A keychain round trip costs microseconds against work that
|
||||
* makes network calls.
|
||||
*/
|
||||
export async function withReplica<T>(
|
||||
accountId: string,
|
||||
fn: (store: ReplicaStore) => Promise<T> | T,
|
||||
): Promise<T> {
|
||||
const storeDir = getStoreDir();
|
||||
if (!storeDir) {
|
||||
throw new IndexSessionError('The offline replica is not enabled in this deployment.', 404);
|
||||
}
|
||||
return withIndexKey(accountId, async (key) => {
|
||||
const store = ReplicaStore.open({ storeDir, accountId, key });
|
||||
try {
|
||||
return await fn(store);
|
||||
} finally {
|
||||
store.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The JMAP account whose mail is replicated.
|
||||
*
|
||||
* v1 replicates the PRIMARY mail account only. Every primary key already carries
|
||||
* `jmap_account_id`, so adding the delegated/shared accounts a single login also
|
||||
* exposes is inserting rows rather than a migration - JMAP ids are unique only
|
||||
* WITHIN an account, and a schema that merged them would be cross-account leakage
|
||||
* that costs nothing to prevent today and is unfixable later.
|
||||
*/
|
||||
export function primaryMailAccountId(session: JmapSessionInfo): string | null {
|
||||
return accountIdFor(session, CAP_MAIL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Single-flight per local account.
|
||||
*
|
||||
* On `globalThis` rather than in module scope for the same reason
|
||||
* `lib/mail-index/key.ts` keeps its channel there: Next re-evaluates route
|
||||
* modules (dev HMR, and separate module instances across route bundles), so a
|
||||
* module-scoped map is not once-per-process and two overlapping requests would
|
||||
* each get their own "single" flight. A Symbol key on globalThis is the one place
|
||||
* in a Node process that survives module re-evaluation.
|
||||
*/
|
||||
const FLIGHT_KEY = Symbol.for('vncmail.offlineReplica.inFlight');
|
||||
|
||||
function flights(): Map<string, Promise<CycleReport>> {
|
||||
const holder = globalThis as unknown as Record<symbol, Map<string, Promise<CycleReport>> | undefined>;
|
||||
const existing = holder[FLIGHT_KEY];
|
||||
if (existing) return existing;
|
||||
const created = new Map<string, Promise<CycleReport>>();
|
||||
holder[FLIGHT_KEY] = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
export interface SyncOptions {
|
||||
/** Overrides the persisted policy for this cycle, and persists the override. */
|
||||
policy?: RetentionPolicy;
|
||||
/** Forces a rebuild: sets the sticky resync flag before the cycle runs. */
|
||||
forceResync?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one cycle for the calling session's account, coalescing concurrent callers
|
||||
* onto the same promise.
|
||||
*
|
||||
* Coalescing rather than aborting is deliberate: an implementation that set an
|
||||
* abort flag and returned produced a cancelled sync and no new one - a "Sync now"
|
||||
* tap during a sync did nothing at all. The in-flight promise is assigned to the
|
||||
* map BEFORE the cycle body runs, because several early-return paths resolve
|
||||
* synchronously and a later assignment leaves a re-entrancy hole; the cleanup is
|
||||
* identity-checked so a slow loser cannot delete a newer flight.
|
||||
*/
|
||||
export async function syncAccount(
|
||||
indexSession: IndexSession,
|
||||
options: SyncOptions = {},
|
||||
): Promise<CycleReport> {
|
||||
const map = flights();
|
||||
const existing = map.get(indexSession.accountId);
|
||||
if (existing) return existing;
|
||||
|
||||
const run = (async (): Promise<CycleReport> => {
|
||||
// The session fetch is the FIRST network call of a cycle, so when the backend
|
||||
// is unreachable this is where it fails - and it must be classified by the same
|
||||
// taxonomy as everything else. Found by execution: without this, an offline
|
||||
// sync surfaced a bare `JmapIndexError` 502 with no error class, so a caller
|
||||
// could not tell "the network is down, retry later" from "this deployment is
|
||||
// broken". "Offline is not an error" has to hold at the very first call too.
|
||||
let session: JmapSessionInfo;
|
||||
try {
|
||||
session = await fetchJmapSession(indexSession.serverUrl, indexSession.authHeader);
|
||||
} catch (error) {
|
||||
const status = (error as { status?: number } | null)?.status;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
// `JmapIndexError.status` is OUR OWN value, not a server response status:
|
||||
// it is 401 for auth, 429 for rate limiting, 504 for a timeout, and 502 for
|
||||
// everything else - INCLUDING a `fetch` rejection with no server involved at
|
||||
// all. So a bare 502 must be classified from the message, or "the machine is
|
||||
// offline" is misread as "the server returned a 5xx". Passing the synthetic
|
||||
// status straight into `classify` produced exactly that, found by the
|
||||
// network-cut integration run.
|
||||
if (status === 401 || status === 403) throw error;
|
||||
const cls =
|
||||
status === 429 ? 'RateLimit' as const
|
||||
: status === 504 ? 'Transport' as const
|
||||
: classify({ message });
|
||||
throw new ReplicaSyncError(cls, message);
|
||||
}
|
||||
const jmapAccountId = primaryMailAccountId(session);
|
||||
if (!jmapAccountId) {
|
||||
throw new IndexSessionError('This account has no JMAP mail capability.', 409);
|
||||
}
|
||||
|
||||
return withReplica(indexSession.accountId, async (store) => {
|
||||
const now = Date.now();
|
||||
if (options.policy) store.transaction(() => { store.setPolicy(options.policy as RetentionPolicy); });
|
||||
if (options.forceResync) {
|
||||
store.transaction(() => { store.patchFlags(now, { resyncRequired: true }); });
|
||||
}
|
||||
const policy = store.getPolicy();
|
||||
const report = await runCycle({
|
||||
store,
|
||||
session,
|
||||
authHeader: indexSession.authHeader,
|
||||
jmapAccountId,
|
||||
policy,
|
||||
now,
|
||||
deadline: now + BUDGET.wallClockMs,
|
||||
});
|
||||
logger.info('offline-replica: cycle complete', {
|
||||
slot: indexSession.slot,
|
||||
ok: report.ok,
|
||||
phase: report.coveragePhase,
|
||||
envelopes: report.envelopesWritten,
|
||||
bodies: report.bodiesWritten,
|
||||
deleted: report.envelopesDeleted,
|
||||
unfinished: report.unfinishedWork,
|
||||
warnings: report.warnings.length,
|
||||
durationMs: report.durationMs,
|
||||
});
|
||||
return report;
|
||||
});
|
||||
})();
|
||||
|
||||
map.set(indexSession.accountId, run);
|
||||
try {
|
||||
return await run;
|
||||
} finally {
|
||||
if (map.get(indexSession.accountId) === run) map.delete(indexSession.accountId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the JMAP account id for a READ without any network call.
|
||||
*
|
||||
* The read path must work with the backend unreachable, so it cannot fetch a JMAP
|
||||
* session to learn the primary account id - that fetch is exactly what fails when
|
||||
* offline. The store knows which account ids it holds rows for; with one
|
||||
* replicated account that is unambiguous, and the caller may also pass an explicit
|
||||
* id.
|
||||
*/
|
||||
export function resolveReadAccountId(store: ReplicaStore, requested?: string | null): string | null {
|
||||
const known = store.knownJmapAccountIds();
|
||||
if (requested && known.includes(requested)) return requested;
|
||||
if (known.length === 1) return known[0];
|
||||
if (requested) return null;
|
||||
return known[0] ?? null;
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
// The error taxonomy. The whole point of this file is one column of one table:
|
||||
// **exactly one error class moves the cursor**, and its action is a full
|
||||
// verified rebuild. Everywhere else, failure means the cursor stands still.
|
||||
//
|
||||
// That is what makes "a failure never causes silent data loss" structural rather
|
||||
// than aspirational - and it is precisely what the mobile client's shipped
|
||||
// defect D4 got wrong, by collapsing every error to `null` and then adopting a
|
||||
// snapshot state as the next cursor. A transient 503 on `Email/changes` was
|
||||
// enough to fast-forward the cursor over every change the client had not seen.
|
||||
|
||||
export type ErrorClass =
|
||||
| 'Transport'
|
||||
| 'RateLimit'
|
||||
| 'ServerTransient'
|
||||
| 'RequestLimit'
|
||||
| 'Auth'
|
||||
| 'Fatal'
|
||||
| 'StateInvalid';
|
||||
|
||||
/** True for the one class that moves a cursor - and it moves it to "invalidated". */
|
||||
export function movesCursor(cls: ErrorClass): boolean {
|
||||
return cls === 'StateInvalid';
|
||||
}
|
||||
|
||||
/**
|
||||
* Only a size/availability problem is worth escalating to a rebuild.
|
||||
*
|
||||
* Escalating on RateLimit would mean the response to a rate-limited server is to
|
||||
* issue far MORE requests - a full window re-enumeration. Escalating on Auth
|
||||
* would let a 401 trigger a rebuild; on Transport, a flaky tunnel would do the
|
||||
* same. Fatal is our own bug and a rebuild will not fix it.
|
||||
*/
|
||||
export function escalationApplies(cls: ErrorClass): boolean {
|
||||
return cls === 'ServerTransient' || cls === 'RequestLimit';
|
||||
}
|
||||
|
||||
/** JMAP method-level error types that invalidate a `/changes` cursor. */const STATE_INVALID_TYPES = new Set(['cannotCalculateChanges']);
|
||||
|
||||
const FATAL_TYPES = new Set([
|
||||
'invalidArguments', 'unknownMethod', 'accountNotFound', 'forbidden',
|
||||
'unsupportedFilter', 'unsupportedSort', 'invalidResultReference',
|
||||
'accountNotSupportedByMethod', 'accountReadOnly',
|
||||
]);
|
||||
|
||||
const REQUEST_LIMIT_TYPES = new Set([
|
||||
'maxSizeRequest', 'maxCallsInRequest', 'requestTooLarge', 'maxObjectsInGet',
|
||||
'tooLarge',
|
||||
]);
|
||||
|
||||
const SERVER_TRANSIENT_TYPES = new Set([
|
||||
'serverUnavailable', 'serverFail', 'serverPartialFail', 'stateMismatch',
|
||||
]);
|
||||
|
||||
export class ReplicaSyncError extends Error {
|
||||
readonly cls: ErrorClass;
|
||||
readonly retryAfterMs?: number;
|
||||
constructor(cls: ErrorClass, message: string, retryAfterMs?: number) {
|
||||
super(message);
|
||||
this.name = 'ReplicaSyncError';
|
||||
this.cls = cls;
|
||||
this.retryAfterMs = retryAfterMs;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Classification is STRUCTURE BEFORE STRINGS: HTTP status, then JMAP error type,
|
||||
* and only then message prose. A method error's `description` can legitimately
|
||||
* contain the words "timeout" or "socket", and `fetch failed: ECONNRESET` must
|
||||
* not be read as a JMAP method error.
|
||||
*/
|
||||
export function classify(input: {
|
||||
httpStatus?: number;
|
||||
jmapErrorType?: string;
|
||||
message?: string;
|
||||
}): ErrorClass {
|
||||
const { httpStatus, jmapErrorType, message } = input;
|
||||
|
||||
if (typeof httpStatus === 'number') {
|
||||
if (httpStatus === 401 || httpStatus === 403) return 'Auth';
|
||||
if (httpStatus === 429) return 'RateLimit';
|
||||
if (httpStatus === 413) return 'RequestLimit';
|
||||
if (httpStatus >= 500) return 'ServerTransient';
|
||||
}
|
||||
|
||||
if (jmapErrorType) {
|
||||
if (STATE_INVALID_TYPES.has(jmapErrorType)) return 'StateInvalid';
|
||||
if (REQUEST_LIMIT_TYPES.has(jmapErrorType)) return 'RequestLimit';
|
||||
if (FATAL_TYPES.has(jmapErrorType)) return 'Fatal';
|
||||
if (SERVER_TRANSIENT_TYPES.has(jmapErrorType)) return 'ServerTransient';
|
||||
if (jmapErrorType === 'limit') return 'RateLimit';
|
||||
// An UNRECOGNISED method-level error is ServerTransient, never Fatal and
|
||||
// never StateInvalid. Guessing transient costs a retry; guessing
|
||||
// state-invalid costs a full resync; guessing fatal stalls the account. The
|
||||
// cheapest wrong answer wins the default.
|
||||
return 'ServerTransient';
|
||||
}
|
||||
|
||||
if (message) {
|
||||
const lower = message.toLowerCase();
|
||||
if (
|
||||
lower.includes('fetch failed') || lower.includes('econnrefused') ||
|
||||
lower.includes('econnreset') || lower.includes('enotfound') ||
|
||||
lower.includes('etimedout') || lower.includes('socket') ||
|
||||
lower.includes('network') || lower.includes('timed out') ||
|
||||
lower.includes('eai_again') || lower.includes('ehostunreach') ||
|
||||
lower.includes('enetunreach') || lower.includes('certificate')
|
||||
) {
|
||||
// "Offline is not an error." Transport failures leave every cursor exactly
|
||||
// where it was and are retried later.
|
||||
return 'Transport';
|
||||
}
|
||||
}
|
||||
|
||||
return 'ServerTransient';
|
||||
}
|
||||
|
||||
/** Full-jitter exponential backoff. */
|
||||
export function backoffDelayMs(attempt: number, opts: { baseMs?: number; capMs?: number } = {}): number {
|
||||
const base = opts.baseMs ?? 1_000;
|
||||
const cap = opts.capMs ?? 60_000;
|
||||
const ceiling = Math.min(cap, base * 2 ** Math.max(0, attempt));
|
||||
// Jitter is not decoration: several triggers fire at once (launch catch-up,
|
||||
// network recovery, a push burst) against one Stalwart instance, which is
|
||||
// exactly the shape that produces a synchronised stampede.
|
||||
return Math.floor(Math.random() * ceiling);
|
||||
}
|
||||
|
||||
/**
|
||||
* The `maxChanges` ladder, monotonically SHRINKING, every rung expressed
|
||||
* relative to rung 0.
|
||||
*
|
||||
* Two bugs live here historically. First, an unbounded middle rung produced a
|
||||
* retry strictly LARGER than the attempt that just failed - actively worsening a
|
||||
* "response too large" error. Then clamping only rung 0 reintroduced it in a
|
||||
* narrower form: a server advertising `maxObjectsInGet: 100` gave rung 0 = 100
|
||||
* and rung 1 = 250. Deriving every rung from rung 0 is what makes
|
||||
* monotonic-non-increase true for every server value.
|
||||
*/
|
||||
export function rungValue(rung: 0 | 1 | 2 | 3, maxObjectsInGet: number | undefined): number {
|
||||
const rung0 = Math.max(1, Math.min(maxObjectsInGet ?? 500, 500));
|
||||
switch (rung) {
|
||||
case 0: return rung0;
|
||||
case 1: return Math.max(1, Math.min(rung0, 250));
|
||||
case 2: return Math.max(1, Math.min(rung0, 50));
|
||||
case 3: return Math.max(1, Math.min(rung0, 25));
|
||||
}
|
||||
}
|
||||
|
||||
export function nextRung(rung: 0 | 1 | 2 | 3): 0 | 1 | 2 | 3 {
|
||||
return rung >= 3 ? 3 : ((rung + 1) as 0 | 1 | 2 | 3);
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
// The replica's JMAP calls. Reuses `lib/mail-index/jmap.ts`'s session fetch,
|
||||
// origin pinning and request plumbing rather than duplicating them (that file
|
||||
// already handles Stalwart's 307 on /.well-known/jmap and refuses to send
|
||||
// credentials off-origin), and adds the delta-sync methods the index never
|
||||
// needed: `Mailbox/changes`, `Email/changes`, the ascending coverage query, and
|
||||
// the two-tier `Email/get`.
|
||||
//
|
||||
// THIS FILE IS THE ONLY PLACE ALLOWED TO MINT A BRANDED STATE TOKEN. That is what
|
||||
// makes cursor provenance checkable by grep: `asChangesState` appears only in the
|
||||
// `/changes` parser, `asSnapshotState` only in the `Foo/get {ids: []}` parser.
|
||||
|
||||
import type { Email, Mailbox } from '@/lib/jmap/types';
|
||||
import { CAP_CORE, CAP_MAIL, jmapRequest, type JmapSessionInfo } from '@/lib/mail-index/jmap';
|
||||
import type { ChangesPage } from './apply';
|
||||
import { ReplicaSyncError, classify } from './errors';
|
||||
import { asChangesState, asSnapshotState, type SnapshotState } from './states';
|
||||
|
||||
/** The envelope tier. Mirrors `lib/jmap/client.ts`'s EMAIL_LIST_PROPERTIES exactly. */
|
||||
export const ENVELOPE_PROPERTIES = [
|
||||
'id', 'threadId', 'mailboxIds', 'keywords', 'size', 'receivedAt',
|
||||
'from', 'to', 'cc', 'subject', 'preview', 'hasAttachment', 'blobId',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* The body tier - everything `lib/jmap/client.ts`'s `getEmail()` asks for beyond
|
||||
* the envelope tier, so a replica-served message is field-for-field what the
|
||||
* online read path produces. `components/email/email-viewer.tsx` reads
|
||||
* `bodyValues` keyed by the SAME partIds as `htmlBody`/`textBody`, so all three
|
||||
* must travel together or the viewer sits on its loading skeleton forever.
|
||||
*/
|
||||
export const BODY_PROPERTIES = [
|
||||
'id', 'sentAt', 'bcc', 'replyTo', 'textBody', 'htmlBody', 'bodyValues',
|
||||
'attachments', 'messageId', 'inReplyTo', 'references', 'headers', 'bodyStructure',
|
||||
] as const;
|
||||
|
||||
/** The three MUTABLE properties. */
|
||||
export const MUTABLE_PROPERTIES = ['id', 'keywords', 'mailboxIds'] as const;
|
||||
|
||||
export const MAX_BODY_VALUE_BYTES = 512_000;
|
||||
|
||||
interface MethodError {
|
||||
type?: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
function asMethodError(args: Record<string, unknown>): MethodError {
|
||||
return {
|
||||
type: typeof args.type === 'string' ? args.type : undefined,
|
||||
description: typeof args.description === 'string' ? args.description : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one JMAP request and classifies any failure. Wraps the shared transport so
|
||||
* a transport-level failure becomes `Transport` (cursor untouched) rather than an
|
||||
* opaque throw the caller has to guess about.
|
||||
*/
|
||||
async function call(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
methodCalls: ReadonlyArray<[string, Record<string, unknown>, string]>,
|
||||
): Promise<Array<[string, Record<string, unknown>, string]>> {
|
||||
try {
|
||||
return await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], methodCalls);
|
||||
} catch (error) {
|
||||
const status = (error as { status?: number } | null)?.status;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new ReplicaSyncError(classify({ httpStatus: status, message }), message);
|
||||
}
|
||||
}
|
||||
|
||||
function findResponse(
|
||||
responses: Array<[string, Record<string, unknown>, string]>,
|
||||
callId: string,
|
||||
): { name: string; args: Record<string, unknown> } | null {
|
||||
for (const [name, args, id] of responses) {
|
||||
if (id === callId) return { name, args };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Turns a method-level `error` response into a classified throw. */
|
||||
function raiseMethodError(args: Record<string, unknown>, context: string): never {
|
||||
const { type, description } = asMethodError(args);
|
||||
throw new ReplicaSyncError(
|
||||
classify({ jmapErrorType: type, message: description }),
|
||||
`${context} failed: ${type ?? 'unknown'}${description ? ` (${description})` : ''}`,
|
||||
);
|
||||
}
|
||||
|
||||
function strArray(v: unknown): string[] {
|
||||
return Array.isArray(v) ? v.filter((x): x is string => typeof x === 'string') : [];
|
||||
}
|
||||
|
||||
// ── snapshot states, for bootstrap / reconcile ───────────────────────────────
|
||||
|
||||
/**
|
||||
* Captures both cursors in ONE request, before touching any data.
|
||||
*
|
||||
* `Foo/get {ids: []}` returns the account's current state token with no records,
|
||||
* which RFC 8620 s5.1 defines as a valid `sinceState` for `Foo/changes`. This is
|
||||
* step 1 of the mandatory bootstrap order and the single thing most likely to be
|
||||
* "optimised" into a permanent data hole: the cursor must be captured BEFORE the
|
||||
* enumeration, so it is deliberately OLDER than the data and the first delta cycle
|
||||
* re-delivers a few changes we already have. The cheaper opposite order - enumerate,
|
||||
* then capture - silently loses every change that arrived during the scan, which on
|
||||
* a large mailbox is minutes.
|
||||
*/
|
||||
export async function captureSnapshotStates(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
): Promise<{ mailbox: SnapshotState; email: SnapshotState }> {
|
||||
const responses = await call(session, authHeader, [
|
||||
['Mailbox/get', { accountId, ids: [] }, 'm'],
|
||||
['Email/get', { accountId, ids: [] }, 'e'],
|
||||
]);
|
||||
const mailbox = findResponse(responses, 'm');
|
||||
const email = findResponse(responses, 'e');
|
||||
if (!mailbox || mailbox.name === 'error') {
|
||||
raiseMethodError(mailbox?.args ?? {}, 'Mailbox/get (state capture)');
|
||||
}
|
||||
if (!email || email.name === 'error') {
|
||||
raiseMethodError(email?.args ?? {}, 'Email/get (state capture)');
|
||||
}
|
||||
return {
|
||||
mailbox: asSnapshotState(mailbox.args.state),
|
||||
email: asSnapshotState(email.args.state),
|
||||
};
|
||||
}
|
||||
|
||||
// ── /changes ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function parseChangesPage(args: Record<string, unknown>): ChangesPage {
|
||||
return {
|
||||
oldState: asChangesState(args.oldState),
|
||||
newState: asChangesState(args.newState),
|
||||
hasMoreChanges: args.hasMoreChanges === true,
|
||||
created: strArray(args.created),
|
||||
updated: strArray(args.updated),
|
||||
destroyed: strArray(args.destroyed),
|
||||
updatedProperties:
|
||||
args.updatedProperties === null
|
||||
? null
|
||||
: Array.isArray(args.updatedProperties)
|
||||
? strArray(args.updatedProperties)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getMailboxChanges(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
sinceState: string,
|
||||
maxChanges: number,
|
||||
): Promise<ChangesPage> {
|
||||
const responses = await call(session, authHeader, [
|
||||
['Mailbox/changes', { accountId, sinceState, maxChanges }, 'c'],
|
||||
]);
|
||||
const res = findResponse(responses, 'c');
|
||||
if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Mailbox/changes');
|
||||
return parseChangesPage(res.args);
|
||||
}
|
||||
|
||||
export async function getEmailChanges(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
sinceState: string,
|
||||
maxChanges: number,
|
||||
): Promise<ChangesPage> {
|
||||
const responses = await call(session, authHeader, [
|
||||
['Email/changes', { accountId, sinceState, maxChanges }, 'c'],
|
||||
]);
|
||||
const res = findResponse(responses, 'c');
|
||||
if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Email/changes');
|
||||
// `Email/changes` has no `updatedProperties` - RFC 8621 s4.3 is a plain
|
||||
// /changes - which is why the 3-property `Email/get` is unavoidable there.
|
||||
return parseChangesPage(res.args);
|
||||
}
|
||||
|
||||
// ── gets ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function getMailboxes(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[] | null,
|
||||
properties?: readonly string[],
|
||||
): Promise<Mailbox[]> {
|
||||
const args: Record<string, unknown> = { accountId, ids: ids === null ? null : [...ids] };
|
||||
if (properties) args.properties = [...properties, 'id'];
|
||||
const responses = await call(session, authHeader, [['Mailbox/get', args, 'g']]);
|
||||
const res = findResponse(responses, 'g');
|
||||
if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Mailbox/get');
|
||||
return Array.isArray(res.args.list) ? (res.args.list as Mailbox[]) : [];
|
||||
}
|
||||
|
||||
export interface EmailGetResult {
|
||||
list: Email[];
|
||||
/** Normal, not an error: the record was destroyed between /changes and /get. */
|
||||
notFound: string[];
|
||||
}
|
||||
|
||||
export async function getEmails(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
tier: 'envelope' | 'mutable' | 'body',
|
||||
): Promise<EmailGetResult> {
|
||||
if (ids.length === 0) return { list: [], notFound: [] };
|
||||
const args: Record<string, unknown> = { accountId, ids: [...ids] };
|
||||
if (tier === 'envelope') {
|
||||
args.properties = [...ENVELOPE_PROPERTIES];
|
||||
} else if (tier === 'mutable') {
|
||||
args.properties = [...MUTABLE_PROPERTIES];
|
||||
} else {
|
||||
args.properties = [...BODY_PROPERTIES];
|
||||
// Without these the bodyValues map comes back EMPTY and every stored body
|
||||
// would be an empty object that renders as a blank message offline.
|
||||
args.fetchTextBodyValues = true;
|
||||
args.fetchHTMLBodyValues = true;
|
||||
args.fetchAllBodyValues = true;
|
||||
args.maxBodyValueBytes = MAX_BODY_VALUE_BYTES;
|
||||
}
|
||||
const responses = await call(session, authHeader, [['Email/get', args, 'g']]);
|
||||
const res = findResponse(responses, 'g');
|
||||
if (!res || res.name === 'error') raiseMethodError(res?.args ?? {}, 'Email/get');
|
||||
return {
|
||||
list: Array.isArray(res.args.list) ? (res.args.list as Email[]) : [],
|
||||
notFound: strArray(res.args.notFound),
|
||||
};
|
||||
}
|
||||
|
||||
// ── coverage enumeration ─────────────────────────────────────────────────────
|
||||
|
||||
export interface CoveragePage {
|
||||
ids: string[];
|
||||
/** Echoed back so the caller can detect the tie-cluster case. */
|
||||
requestedAfter: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The ascending keyset walk.
|
||||
*
|
||||
* ASCENDING is not a style choice. New mail arrives at the TAIL, so insertions
|
||||
* never shift rows the scan has already passed. With a DESCENDING sort and
|
||||
* position-based paging, one delivery between page 1 and page 2 pushes a message
|
||||
* from page 1's boundary into page 2's start and one message out of the scan's
|
||||
* reach entirely - and that message is pre-existing relative to our cursor, so
|
||||
* `Email/changes` will never report it. A permanent hole with no signal it exists.
|
||||
*
|
||||
* `calculateTotal: false` because the total is unstable and unused.
|
||||
*/
|
||||
export async function queryAscending(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
after: string,
|
||||
limit: number,
|
||||
anchor?: { anchor: string; anchorOffset: number },
|
||||
): Promise<CoveragePage> {
|
||||
const args: Record<string, unknown> = {
|
||||
accountId,
|
||||
filter: { after },
|
||||
sort: [{ property: 'receivedAt', isAscending: true }],
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
};
|
||||
if (anchor) {
|
||||
args.anchor = anchor.anchor;
|
||||
args.anchorOffset = anchor.anchorOffset;
|
||||
}
|
||||
const responses = await call(session, authHeader, [['Email/query', args, 'q']]);
|
||||
const res = findResponse(responses, 'q');
|
||||
if (!res || res.name === 'error') {
|
||||
const { type } = asMethodError(res?.args ?? {});
|
||||
if (type === 'anchorNotFound') {
|
||||
// Not a failure - the caller falls back to the last-resort rung.
|
||||
throw new AnchorNotFoundError();
|
||||
}
|
||||
raiseMethodError(res?.args ?? {}, 'Email/query');
|
||||
}
|
||||
return { ids: strArray(res.args.ids), requestedAfter: after };
|
||||
}
|
||||
|
||||
export class AnchorNotFoundError extends Error {
|
||||
constructor() {
|
||||
super('Email/query rejected the anchor');
|
||||
this.name = 'AnchorNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
/** `maxObjectsInGet`, so the maxChanges ladder can be clamped to what the server allows. */
|
||||
export function maxObjectsInGet(session: JmapSessionInfo): number | undefined {
|
||||
const core = session.capabilities[CAP_CORE];
|
||||
if (!core || typeof core !== 'object') return undefined;
|
||||
const value = (core as Record<string, unknown>).maxObjectsInGet;
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
// The offline READ path: stored rows back into the exact `Email` / `Mailbox`
|
||||
// shapes `lib/jmap/client.ts` returns, so the renderer cannot tell the difference.
|
||||
//
|
||||
// COHERENCE (the review's H3, the one finding that genuinely returns once a
|
||||
// replica exists). The webmail already does LOCAL DELTA ARITHMETIC on mailbox
|
||||
// unread counts and totals for mark-read/move/delete, with a comment referencing
|
||||
// a production bug from getting that cutoff wrong. A read-only cache sitting
|
||||
// underneath that arithmetic needs an explicit coherence story, and the story is:
|
||||
//
|
||||
// THE REPLICA IS A FALLBACK, NEVER A CACHE IN FRONT OF THE SERVER.
|
||||
//
|
||||
// It is consulted only after a read has actually failed at the transport level
|
||||
// (see `lib/offline-fallback-client.ts`), so an online session never sees a
|
||||
// replica count and the arithmetic never operates on replica numbers. While
|
||||
// offline, counts are whatever the last successful sync recorded and any local
|
||||
// mark-read drift is bounded, invisible in the same session, and repaired by the
|
||||
// next `Mailbox/changes` - which is the authoritative source for all four
|
||||
// counters. The alternative - serving the replica first and reconciling - is what
|
||||
// would need the coherence rules the review asked for, and is not what this does.
|
||||
//
|
||||
// Every shape here is intentionally what the ONLINE path produces, including
|
||||
// `parseEmailHeaders`' derived security fields, because
|
||||
// `components/email/email-viewer.tsx` reads them directly. In particular
|
||||
// `bodyValues` must be keyed by the same partIds as `htmlBody`/`textBody`, or the
|
||||
// viewer's `isBodyLoading` gate sits on its skeleton forever.
|
||||
|
||||
import { parseAuthenticationResults, parseSpamLLM, parseSpamScore } from '@/lib/email-headers';
|
||||
import type { Email, EmailAddress, Mailbox } from '@/lib/jmap/types';
|
||||
import type { ReplicaStore } from './store';
|
||||
import type { MailboxRow } from './types';
|
||||
|
||||
const DEFAULT_RIGHTS: Mailbox['myRights'] = {
|
||||
mayReadItems: true, mayAddItems: false, mayRemoveItems: false, maySetSeen: false,
|
||||
maySetKeywords: false, mayCreateChild: false, mayRename: false, mayDelete: false,
|
||||
maySubmit: false,
|
||||
};
|
||||
|
||||
function parseJson<T>(raw: unknown, fallback: T): T {
|
||||
if (typeof raw !== 'string' || raw.length === 0) return fallback;
|
||||
try {
|
||||
return JSON.parse(raw) as T;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
export function rowToMailbox(row: MailboxRow): Mailbox {
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
parentId: row.parentId ?? undefined,
|
||||
role: row.role ?? undefined,
|
||||
sortOrder: row.sortOrder ?? 0,
|
||||
totalEmails: row.totalEmails ?? 0,
|
||||
unreadEmails: row.unreadEmails ?? 0,
|
||||
totalThreads: row.totalThreads ?? 0,
|
||||
unreadThreads: row.unreadThreads ?? 0,
|
||||
// Offline, the rights that matter are the read ones. Every mutating right
|
||||
// defaults to false so no UI offers an action that cannot possibly succeed
|
||||
// with no network; the real rights return with the next sync.
|
||||
myRights: parseJson<Mailbox['myRights']>(row.myRightsJson, DEFAULT_RIGHTS),
|
||||
isSubscribed: row.isSubscribed,
|
||||
};
|
||||
}
|
||||
|
||||
/** The envelope tier, as `getEmails()` would return it. */
|
||||
export function rowToEnvelope(row: Record<string, unknown>, mailboxIds: readonly string[]): Email {
|
||||
const keywords = parseJson<Record<string, boolean>>(row.keywords_json, {});
|
||||
const mailboxMap: Record<string, boolean> = {};
|
||||
for (const id of mailboxIds) mailboxMap[id] = true;
|
||||
return {
|
||||
id: String(row.id),
|
||||
threadId: typeof row.thread_id === 'string' ? row.thread_id : String(row.id),
|
||||
mailboxIds: mailboxMap,
|
||||
keywords,
|
||||
size: typeof row.size === 'number' ? row.size : 0,
|
||||
receivedAt: String(row.received_at),
|
||||
from: parseJson<EmailAddress[] | undefined>(row.from_json, undefined),
|
||||
to: parseJson<EmailAddress[] | undefined>(row.to_json, undefined),
|
||||
cc: parseJson<EmailAddress[] | undefined>(row.cc_json, undefined),
|
||||
subject: typeof row.subject === 'string' ? row.subject : undefined,
|
||||
preview: typeof row.preview === 'string' ? row.preview : undefined,
|
||||
hasAttachment: row.has_attachment !== 0,
|
||||
blobId: typeof row.blob_id === 'string' ? row.blob_id : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
interface StoredBody {
|
||||
sentAt?: string;
|
||||
bcc?: EmailAddress[];
|
||||
replyTo?: EmailAddress[];
|
||||
textBody?: Email['textBody'];
|
||||
htmlBody?: Email['htmlBody'];
|
||||
bodyValues?: Email['bodyValues'];
|
||||
attachments?: Email['attachments'];
|
||||
messageId?: string;
|
||||
inReplyTo?: string[];
|
||||
references?: string[];
|
||||
headers?: unknown;
|
||||
bodyStructure?: Email['bodyStructure'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalises JMAP's `headers` array into the record shape the renderer expects
|
||||
* and derives the security fields, reproducing what `JMAPClient`'s private
|
||||
* `parseEmailHeaders` does on the online path.
|
||||
*
|
||||
* Reproduced here rather than imported because `lib/jmap/client.ts` is a
|
||||
* 7400-line renderer object that holds credentials in instance fields, opens push
|
||||
* connections and wires itself into Zustand stores - importing it into a server
|
||||
* route would drag all of that into the server bundle. The parsing HELPERS in
|
||||
* `lib/email-headers` are shared, so the only duplicated logic is the array->record
|
||||
* flattening.
|
||||
*/
|
||||
function applyHeaders(email: Email, rawHeaders: unknown): void {
|
||||
let record: Record<string, string | string[]>;
|
||||
if (Array.isArray(rawHeaders)) {
|
||||
record = {};
|
||||
for (const header of rawHeaders as Array<{ name?: string; value?: string }>) {
|
||||
if (!header?.name || !header?.value) continue;
|
||||
const existing = record[header.name];
|
||||
if (existing) {
|
||||
record[header.name] = Array.isArray(existing)
|
||||
? [...existing, header.value]
|
||||
: [existing, header.value];
|
||||
} else {
|
||||
record[header.name] = header.value;
|
||||
}
|
||||
}
|
||||
} else if (rawHeaders && typeof rawHeaders === 'object') {
|
||||
record = rawHeaders as Record<string, string | string[]>;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
email.headers = record;
|
||||
|
||||
const authResults = record['Authentication-Results'];
|
||||
if (authResults) {
|
||||
const value = Array.isArray(authResults) ? authResults.join('; ') : authResults;
|
||||
email.authenticationResults = parseAuthenticationResults(value);
|
||||
}
|
||||
|
||||
for (const name of ['X-Spam-Score', 'X-Spam-Status', 'X-Spam-Result', 'X-Rspamd-Score']) {
|
||||
const header = record[name];
|
||||
if (!header) continue;
|
||||
const value = Array.isArray(header) ? header[0] : header;
|
||||
const parsed = parseSpamScore(String(value).trim());
|
||||
if (parsed) {
|
||||
email.spamScore = parsed.score;
|
||||
email.spamStatus = parsed.status;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const llm = record['X-Spam-LLM'];
|
||||
if (llm) {
|
||||
const parsed = parseSpamLLM(String(Array.isArray(llm) ? llm[0] : llm));
|
||||
if (parsed) email.spamLLM = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
export interface OfflineMessage {
|
||||
email: Email;
|
||||
/** False when only the envelope is held, so the caller can say so rather than render blank. */
|
||||
hasBody: boolean;
|
||||
}
|
||||
|
||||
/** One full message, envelope + body, exactly as `getEmail()` would return it. */
|
||||
export function readMessage(
|
||||
store: ReplicaStore,
|
||||
jmapAccountId: string,
|
||||
id: string,
|
||||
): OfflineMessage | null {
|
||||
const row = store.getEnvelopeRaw(jmapAccountId, id);
|
||||
if (!row) return null;
|
||||
const email = rowToEnvelope(row, store.mailboxIdsFor(jmapAccountId, id));
|
||||
|
||||
const bodyJson = store.getBody(jmapAccountId, id);
|
||||
if (!bodyJson) return { email, hasBody: false };
|
||||
|
||||
const body = parseJson<StoredBody>(bodyJson, {});
|
||||
email.sentAt = body.sentAt;
|
||||
email.bcc = body.bcc;
|
||||
email.replyTo = body.replyTo;
|
||||
email.textBody = body.textBody;
|
||||
email.htmlBody = body.htmlBody;
|
||||
email.bodyValues = body.bodyValues;
|
||||
email.attachments = body.attachments;
|
||||
email.messageId = body.messageId;
|
||||
email.inReplyTo = body.inReplyTo;
|
||||
email.references = body.references;
|
||||
email.bodyStructure = body.bodyStructure;
|
||||
applyHeaders(email, body.headers);
|
||||
|
||||
// A body row whose `bodyValues` came back empty would render as a blank
|
||||
// message and, worse, leave the viewer's loading gate stuck. Report it as
|
||||
// "envelope only" instead, which the UI can explain.
|
||||
const hasBody =
|
||||
!!email.bodyValues && Object.keys(email.bodyValues).length > 0;
|
||||
return { email, hasBody };
|
||||
}
|
||||
|
||||
export function readMailboxes(store: ReplicaStore, jmapAccountId: string): Mailbox[] {
|
||||
return store.listMailboxes(jmapAccountId).map(rowToMailbox);
|
||||
}
|
||||
|
||||
export function readEnvelopePage(
|
||||
store: ReplicaStore,
|
||||
jmapAccountId: string,
|
||||
mailboxId: string | null,
|
||||
limit: number,
|
||||
offset: number,
|
||||
): { emails: Email[]; total: number; hasMore: boolean } {
|
||||
const { rows, total } = store.listEnvelopes(jmapAccountId, mailboxId, limit, offset);
|
||||
const emails = rows.map((row) =>
|
||||
rowToEnvelope(row, store.mailboxIdsFor(jmapAccountId, String(row.id))),
|
||||
);
|
||||
return { emails, total, hasMore: offset + emails.length < total };
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
// Retention floors, and the clock-jump guard.
|
||||
//
|
||||
// THE BUG THIS FILE IS SHAPED BY. No cursor and no ordering in this engine
|
||||
// depends on the device clock - but the retention BOUNDARY does, so a large clock
|
||||
// skew moves the window. The mobile client added a guard: if the computed floor
|
||||
// moves more than ~25 h from the last one, keep the previous floor and warn.
|
||||
//
|
||||
// The guard then wiped the entire offline store. Mechanism, exactly:
|
||||
//
|
||||
// 1. clock jumps forward a year
|
||||
// 2. cycle N computes a floor a year ahead, detects the >25 h move, holds the
|
||||
// old floor for this cycle - and PERSISTS THE COMPUTED (jumped) FLOOR as
|
||||
// `lastWindowFloor`
|
||||
// 3. the follow-on cycle a few seconds later computes a floor within seconds of
|
||||
// the persisted one, so `delta <= threshold`, so the guard passes
|
||||
// 4. that floor is classified as a retention NARROW, and every envelope below
|
||||
// it is evicted - i.e. all of them, since the floor is a year in the future
|
||||
// 5. `coveredFrom` then claims the range complete, and `/changes` cannot
|
||||
// re-deliver pre-existing mail. Unrecoverable.
|
||||
//
|
||||
// A clock glitch wiped the store about five seconds after being detected,
|
||||
// THROUGH the very mechanism meant to prevent that. Its reproduction returned 0
|
||||
// envelopes from a 4-envelope store.
|
||||
//
|
||||
// The generalisable lesson, worth more than the code: a guard that DETECTS an
|
||||
// anomaly but PERSISTS the anomalous value converts a transient glitch into a
|
||||
// legitimised new baseline. Any "suppress and remember" guard must remember the
|
||||
// value it USED, not the value it rejected - and must expose a separate
|
||||
// "don't delete anything on this basis" bit, because suppressing the floor is not
|
||||
// the same as suppressing the deletions the floor authorises.
|
||||
|
||||
import type { RetentionPolicy } from './store';
|
||||
|
||||
/** A day plus an hour: an ordinary DST shift or NTP correction must not trip it. */
|
||||
export const CLOCK_JUMP_GUARD_MS = 25 * 60 * 60 * 1000;
|
||||
|
||||
export interface RetentionFloors {
|
||||
envelopeFrom: string;
|
||||
bodyFrom: string;
|
||||
maxBodyBytes: number;
|
||||
}
|
||||
|
||||
function isoDaysAgo(now: number, days: number): string {
|
||||
return new Date(now - days * 24 * 60 * 60 * 1000).toISOString();
|
||||
}
|
||||
|
||||
export function computeFloors(policy: RetentionPolicy, now: number): RetentionFloors {
|
||||
// The body window can never be wider than the envelope window: a body with no
|
||||
// envelope is an orphan by construction, and the whole point of two tiers is
|
||||
// envelopes being a superset of bodies.
|
||||
const bodyDays = Math.min(policy.bodyDays, policy.envelopeDays);
|
||||
return {
|
||||
envelopeFrom: isoDaysAgo(now, policy.envelopeDays),
|
||||
bodyFrom: isoDaysAgo(now, bodyDays),
|
||||
maxBodyBytes: Math.max(0, Math.floor(policy.maxBodyMB * 1024 * 1024)),
|
||||
};
|
||||
}
|
||||
|
||||
export interface GuardedFloor {
|
||||
/** The floor to actually use for eviction and for any sweep. */
|
||||
envelopeFrom: string;
|
||||
/** True when the guard suppressed a suspicious jump. */
|
||||
suppressed: boolean;
|
||||
/**
|
||||
* False while the floor in use came from a SUSPECT clock reading. Retention
|
||||
* eviction and the reconcile sweep must BOTH refuse to act on it - suppressing
|
||||
* the floor is not the same as suppressing the deletions it authorises.
|
||||
*/
|
||||
evictionAllowed: boolean;
|
||||
/** What to persist as `lastWindowFloor`. */
|
||||
nextLastWindowFloor: string;
|
||||
warning?: string;
|
||||
}
|
||||
|
||||
export function guardFloorAgainstClockJump(
|
||||
computed: string,
|
||||
lastWindowFloor: string | undefined,
|
||||
opts: { policyChanged?: boolean } = {},
|
||||
): GuardedFloor {
|
||||
const adopt = (floor: string): GuardedFloor => ({
|
||||
envelopeFrom: floor,
|
||||
suppressed: false,
|
||||
evictionAllowed: true,
|
||||
nextLastWindowFloor: floor,
|
||||
});
|
||||
|
||||
if (!lastWindowFloor) return adopt(computed);
|
||||
|
||||
// An explicit `envelopeDays` change is INTENT, not a glitch, and must take
|
||||
// effect - including its eviction. This is also why "adopt on the second
|
||||
// consistent observation" had to go: with intent handled here, that rule
|
||||
// existed ONLY for the clock-anomaly case, i.e. only for the case where
|
||||
// adopting is the harmful thing to do.
|
||||
if (opts.policyChanged) return adopt(computed);
|
||||
|
||||
const delta = Math.abs(Date.parse(computed) - Date.parse(lastWindowFloor));
|
||||
if (!Number.isFinite(delta) || delta <= CLOCK_JUMP_GUARD_MS) return adopt(computed);
|
||||
|
||||
// SUSPECT. Ignore the reading entirely, keep the previous floor, and persist
|
||||
// THE PREVIOUS FLOOR - so every subsequent cycle re-detects the same jump and
|
||||
// stays suppressed. Persisting the computed value here is the wipe described
|
||||
// at the top of this file.
|
||||
//
|
||||
// The trade-off, stated rather than hidden: on a device whose clock is
|
||||
// permanently wrong by more than a day, retention stops tracking the clock and
|
||||
// the store keeps MORE mail than the setting says. That is bounded (envelopes
|
||||
// are ~1 KB, bodies are capped in bytes) and self-clears the moment the user
|
||||
// changes a retention setting or the clock returns. Keeping too much mail is
|
||||
// the correct direction to fail for a feature whose entire purpose is having
|
||||
// mail available offline.
|
||||
return {
|
||||
envelopeFrom: lastWindowFloor,
|
||||
suppressed: true,
|
||||
evictionAllowed: false,
|
||||
nextLastWindowFloor: lastWindowFloor,
|
||||
warning:
|
||||
`retention floor moved ${Math.round(delta / 3_600_000)}h in one step ` +
|
||||
`(${lastWindowFloor} -> ${computed}); treating it as a clock anomaly and ` +
|
||||
`refusing to evict or sweep on this basis`,
|
||||
};
|
||||
}
|
||||
|
||||
export type FloorMovement = 'unchanged' | 'widened' | 'narrowed';
|
||||
|
||||
export function floorMovement(previous: string | undefined, next: string): FloorMovement {
|
||||
if (!previous) return 'unchanged';
|
||||
if (next === previous) return 'unchanged';
|
||||
// A LATER floor keeps less mail.
|
||||
return next > previous ? 'narrowed' : 'widened';
|
||||
}
|
||||
|
||||
export interface WindowAdjustment {
|
||||
/** Set when envelopes below this floor should be evicted. */
|
||||
evictBelow?: string;
|
||||
/** Set when coverage must re-scan from a wider floor. */
|
||||
rescanFrom?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a floor movement asks for.
|
||||
*
|
||||
* A WIDEN is not a resync: `targetFrom` moves back, coverage re-enters
|
||||
* `scanning`, and the cursors are untouched. A NARROW evicts.
|
||||
*/
|
||||
export function adjustForWindow(movement: FloorMovement, floor: string): WindowAdjustment {
|
||||
if (movement === 'narrowed') return { evictBelow: floor };
|
||||
if (movement === 'widened') return { rescanFrom: floor };
|
||||
return {};
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// The gate every replica route shares, and its error mapping.
|
||||
//
|
||||
// 404, not 403, when the feature is absent: the standalone server artifact is the
|
||||
// SAME one the production Dockerfile ships to multi-tenant deployments, where a
|
||||
// server-side replica of every user's mail would be badly wrong. Nothing should
|
||||
// learn the routes exist in a deployment that does not have the feature.
|
||||
|
||||
import { NextResponse } from 'next/server';
|
||||
import { logger } from '@/lib/logger';
|
||||
import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
||||
import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key';
|
||||
import { getStoreDir } from '@/lib/mail-index/paths';
|
||||
import { JmapIndexError } from '@/lib/mail-index/jmap';
|
||||
import { IndexSessionError } from '@/lib/mail-index/reindex';
|
||||
import { ReplicaUnavailableError } from './store';
|
||||
import { ReplicaSyncError } from './errors';
|
||||
|
||||
/** Returns a response to send immediately, or `null` when the gate is open. */
|
||||
export function gateReplicaRoute(): NextResponse | null {
|
||||
if (!getStoreDir()) return new NextResponse(null, { status: 404 });
|
||||
if (!hasKeyChannel()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'The offline replica has no key channel in this process.', code: 'no-key-channel' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
if (!isSqlcipherAvailable()) {
|
||||
// The native binding is an optionalDependency, so "not installed" is a normal
|
||||
// state on a platform with no prebuild - not an error to log loudly.
|
||||
return NextResponse.json(
|
||||
{ error: 'Encrypted local storage is unavailable on this platform.', code: 'no-binding' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function replicaErrorResponse(error: unknown, context: string): NextResponse {
|
||||
if (error instanceof IndexSessionError) {
|
||||
return NextResponse.json({ error: error.message }, { status: error.status });
|
||||
}
|
||||
if (error instanceof JmapIndexError) {
|
||||
return NextResponse.json({ error: error.message }, { status: error.status });
|
||||
}
|
||||
if (error instanceof IndexKeyError) {
|
||||
// `no-secure-storage` is the Linux-without-a-keyring refusal: a real, expected
|
||||
// outcome with a user-facing explanation, not a server fault.
|
||||
const status = error.code === 'no-secure-storage' ? 503 : 500;
|
||||
return NextResponse.json({ error: error.message, code: error.code }, { status });
|
||||
}
|
||||
if (error instanceof ReplicaUnavailableError) {
|
||||
return NextResponse.json({ error: error.message, code: 'unavailable' }, { status: 503 });
|
||||
}
|
||||
if (error instanceof ReplicaSyncError) {
|
||||
// A transport failure here means the BACKEND is unreachable, which for a sync
|
||||
// is an expected outcome rather than a server fault - 503 with the class, so
|
||||
// the renderer can retry rather than surface an error.
|
||||
const status = error.cls === 'Auth' ? 401 : error.cls === 'RateLimit' ? 429 : 503;
|
||||
return NextResponse.json({ error: error.message, code: error.cls }, { status });
|
||||
}
|
||||
logger.error(`offline-replica: ${context} failed`, {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return NextResponse.json({ error: `${context} failed` }, { status: 500 });
|
||||
}
|
||||
|
||||
export const NO_STORE = { 'Cache-Control': 'no-store' } as const;
|
||||
@@ -0,0 +1,178 @@
|
||||
// The offline mail replica's schema.
|
||||
//
|
||||
// WHY IT LIVES IN THE SAME ENCRYPTED FILE AS THE SEARCH INDEX
|
||||
// (`lib/mail-index/paths.ts`'s `indexDbPath`), on its own connection:
|
||||
//
|
||||
// * One encryption boundary, one key, one keychain entry, one purge. A second
|
||||
// keystore would double the number of places a key can be mishandled for no
|
||||
// gain - and `electron/key-service.ts` + the fd-3 channel already work.
|
||||
// * `sync_state` (cursors, coverage, flags) sits in the SAME FILE as the
|
||||
// records it describes. That is load-bearing, not tidiness: deleting the file
|
||||
// removes cursors and records together, so a cursor can never survive a wipe
|
||||
// and then be advanced over changes that will never be re-delivered. A
|
||||
// cursor in a sidecar JSON file is exactly the class of bug the mobile
|
||||
// client's S1 finding is about.
|
||||
// * The tables are DISJOINT from the index's (`doc`, `doc_fts`), so the two
|
||||
// subsystems never contend for a row - only, briefly, for SQLite's write
|
||||
// lock, which `PRAGMA busy_timeout` resolves. Both open with WAL.
|
||||
//
|
||||
// The one coupling to accept: `lib/mail-index/store.ts` drops `meta` on an index
|
||||
// schema bump, which takes `replica_schema_version` with it. The replica reads
|
||||
// that as "version missing" and purges + re-bootstraps - correct, just wasteful,
|
||||
// and only on an index schema change. What must NOT happen is records surviving
|
||||
// while the version row vanishes, which is why the purge below is all-or-nothing
|
||||
// and includes `replica_sync_state`.
|
||||
//
|
||||
// Deliberately NO foreign keys from `replica_email_mailbox.mailbox_id` ->
|
||||
// `replica_mailbox`, and no cascade from `replica_envelope`. The two change
|
||||
// streams are not transactionally coupled, so a membership row referencing a
|
||||
// not-yet-fetched or already-destroyed mailbox is a NORMAL transient state; an
|
||||
// FK would turn correct behaviour into a constraint violation, and a cascade on
|
||||
// mailbox deletion would delete mail, violating the deletion-provenance rule
|
||||
// (only the Email stream may delete an email).
|
||||
|
||||
/** Bumped on any incompatible change. Mismatch = purge + re-bootstrap, never migrate. */
|
||||
export const REPLICA_SCHEMA_VERSION = 1;
|
||||
|
||||
export const REPLICA_VERSION_KEY = 'replica_schema_version';
|
||||
|
||||
export const REPLICA_DDL = `
|
||||
CREATE TABLE IF NOT EXISTS replica_mailbox (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
parent_id TEXT,
|
||||
role TEXT,
|
||||
sort_order INTEGER,
|
||||
total_emails INTEGER,
|
||||
unread_emails INTEGER,
|
||||
total_threads INTEGER,
|
||||
unread_threads INTEGER,
|
||||
my_rights_json TEXT,
|
||||
is_subscribed INTEGER,
|
||||
PRIMARY KEY (jmap_account_id, id)
|
||||
);
|
||||
|
||||
-- The envelope tier: everything EMAIL_LIST_PROPERTIES carries, so an offline
|
||||
-- message list renders exactly as an online one does.
|
||||
CREATE TABLE IF NOT EXISTS replica_envelope (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
thread_id TEXT,
|
||||
received_at TEXT NOT NULL,
|
||||
size INTEGER,
|
||||
subject TEXT,
|
||||
preview TEXT,
|
||||
from_json TEXT,
|
||||
to_json TEXT,
|
||||
cc_json TEXT,
|
||||
blob_id TEXT,
|
||||
has_attachment INTEGER NOT NULL DEFAULT 0,
|
||||
keywords_json TEXT NOT NULL DEFAULT '{}',
|
||||
-- Owned by the BODY tier. Excluded from the envelope upsert's DO UPDATE SET,
|
||||
-- or an idempotent page replay would look like "body missing" to the backfill
|
||||
-- job and re-download every body in the page.
|
||||
has_body INTEGER NOT NULL DEFAULT 0,
|
||||
body_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
cached_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (jmap_account_id, id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS replica_envelope_received
|
||||
ON replica_envelope(jmap_account_id, received_at DESC);
|
||||
-- The body-backfill driver: envelopes inside the body window with no body yet.
|
||||
CREATE INDEX IF NOT EXISTS replica_envelope_nobody
|
||||
ON replica_envelope(jmap_account_id, has_body, received_at DESC);
|
||||
|
||||
-- Membership is its own table: an email is in many mailboxes, and listing by
|
||||
-- folder must be an index seek rather than a scan of every cached row.
|
||||
CREATE TABLE IF NOT EXISTS replica_email_mailbox (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
email_id TEXT NOT NULL,
|
||||
mailbox_id TEXT NOT NULL,
|
||||
PRIMARY KEY (jmap_account_id, email_id, mailbox_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS replica_email_mailbox_by_mailbox
|
||||
ON replica_email_mailbox(jmap_account_id, mailbox_id);
|
||||
|
||||
-- The body tier. "received_at" is duplicated here on purpose so cap eviction is
|
||||
-- a single-table ordered scan that cannot be blinded by a missing join.
|
||||
CREATE TABLE IF NOT EXISTS replica_body (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
email_id TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
json TEXT NOT NULL,
|
||||
bytes INTEGER NOT NULL,
|
||||
PRIMARY KEY (jmap_account_id, email_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS replica_body_received
|
||||
ON replica_body(jmap_account_id, received_at ASC);
|
||||
|
||||
-- "gave_up" is what makes a body-tier TERMINAL STATE durable, and it is the fix
|
||||
-- for the worst bug found on the mobile client. Deleting the queue row on
|
||||
-- give-up was not enough: the backfill job's driver is "envelope with no body",
|
||||
-- a predicate that CANNOT distinguish "not fetched yet" from "deliberately not
|
||||
-- kept". So the next pass re-inserted a fresh attempts=0 row and a
|
||||
-- permanently-failing body was retried five times per cycle, forever. Same
|
||||
-- shape for a "notFound" body, and worst of all for a body shed by the size cap:
|
||||
-- shed -> still inside the body window -> re-enqueued -> re-downloaded -> shed
|
||||
-- again. Unbounded data use with no termination. Keeping the row with a flag is
|
||||
-- what closes all three.
|
||||
CREATE TABLE IF NOT EXISTS replica_body_queue (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
email_id TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at INTEGER,
|
||||
last_error TEXT,
|
||||
gave_up INTEGER NOT NULL DEFAULT 0,
|
||||
gave_up_reason TEXT,
|
||||
PRIMARY KEY (jmap_account_id, email_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS replica_body_queue_wanted
|
||||
ON replica_body_queue(jmap_account_id, gave_up, received_at DESC);
|
||||
|
||||
-- Cursors, coverage and flags. Row-per-field, NOT one JSON blob: a blob loaded
|
||||
-- at cycle start and written at cycle end silently reverts any concurrent write
|
||||
-- to a different field. On the mobile client that produced an empty record store
|
||||
-- with a live advanced cursor and resyncRequired reset to false - a permanent
|
||||
-- silent data gap that is unreachable by design.
|
||||
CREATE TABLE IF NOT EXISTS replica_sync_state (
|
||||
k TEXT PRIMARY KEY,
|
||||
v TEXT NOT NULL
|
||||
);
|
||||
`;
|
||||
|
||||
/**
|
||||
* Everything a purge removes. `replica_sync_state` is INCLUDED: a record wipe
|
||||
* that leaves cursors behind is the one state from which no amount of syncing
|
||||
* recovers, because `/changes` structurally cannot re-deliver mail that already
|
||||
* existed when the cursor was captured.
|
||||
*/
|
||||
export const REPLICA_TABLES: readonly string[] = [
|
||||
'replica_envelope',
|
||||
'replica_email_mailbox',
|
||||
'replica_body',
|
||||
'replica_body_queue',
|
||||
'replica_mailbox',
|
||||
'replica_sync_state',
|
||||
];
|
||||
|
||||
/** Record tables only - used when a reconcile rebuilds without discarding policy. */
|
||||
export const REPLICA_RECORD_TABLES: readonly string[] = [
|
||||
'replica_envelope',
|
||||
'replica_email_mailbox',
|
||||
'replica_body',
|
||||
'replica_body_queue',
|
||||
'replica_mailbox',
|
||||
];
|
||||
|
||||
export const FLAGS_KEY = 'flags';
|
||||
export const POLICY_KEY = 'policy';
|
||||
|
||||
export function cursorStateKey(jmapAccountId: string, type: string): string {
|
||||
return `cursor:${jmapAccountId}:${type}`;
|
||||
}
|
||||
|
||||
export function coverageStateKey(jmapAccountId: string): string {
|
||||
return `coverage:${jmapAccountId}`;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Cursor provenance: the type-level machinery that makes "never adopt an
|
||||
// `Email/get` state as an `Email/changes` cursor" a compile error rather than a
|
||||
// code-review convention.
|
||||
//
|
||||
// This exact bug shipped on the mobile client (its defect D4) and silently
|
||||
// corrupted sync: `getEmailChanges` returned `null` for ANY error, so a
|
||||
// transient 503 on `Email/changes` caused an `Email/get` state captured in the
|
||||
// same cycle to be adopted as the next cursor - fast-forwarding the cursor over
|
||||
// every change the client had not seen, with no resync. The cost is invisible:
|
||||
// the store looks healthy and is permanently missing mail.
|
||||
//
|
||||
// Two brands, and an ORDERING rule rather than a source rule. "Only a
|
||||
// `Foo/changes.newState` may ever be a cursor" is tempting but FALSE - bootstrap
|
||||
// and reconcile legitimately seed from `Foo/get {ids: []}`'s `state`, which
|
||||
// RFC 8620 s5.1 explicitly permits. A rule the design itself has to violate is a
|
||||
// rule that gets bypassed at the one call site that matters, so the rule is:
|
||||
//
|
||||
// A cursor ADVANCES to a ChangesState from the same (jmapAccountId, type).
|
||||
// It may be SEEDED from a SnapshotState only inside an EnumerationCommitment
|
||||
// whose enumeration starts after that snapshot. Nothing else, from anywhere,
|
||||
// ever becomes a cursor.
|
||||
|
||||
/** Types we hold a `/changes` cursor for. NOT a list of push types. */
|
||||
export type CursorType = 'Email' | 'Mailbox';
|
||||
|
||||
export const CURSOR_TYPES: readonly CursorType[] = ['Email', 'Mailbox'];
|
||||
|
||||
/** From a `Foo/changes` response's `newState`. The only value the delta path may advance to. */
|
||||
export type ChangesState = string & { readonly __brand: 'ChangesState' };
|
||||
|
||||
/** From a `Foo/get` response's `state`. A valid cursor ONLY under the ordering rule above. */
|
||||
export type SnapshotState = string & { readonly __brand: 'SnapshotState' };
|
||||
|
||||
/**
|
||||
* A JMAP body is parsed JSON, so without a runtime check a `null`, a number or
|
||||
* an object could be laundered through a cast into something the engine treats
|
||||
* as a cursor forever. The brand certifies PROVENANCE; this certifies SHAPE.
|
||||
*/
|
||||
function certifyStateToken(value: unknown, kind: string): string {
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new TypeError(
|
||||
`${kind}: expected a non-empty string state token, got ` +
|
||||
`${value === null ? 'null' : typeof value}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint a `ChangesState`. Callable ONLY from the `Foo/changes` response parser in
|
||||
* `./jmap.ts` - that is the entire point of the brand. There is a test asserting
|
||||
* no other module casts to these types.
|
||||
*/
|
||||
export function asChangesState(newState: unknown): ChangesState {
|
||||
return certifyStateToken(newState, 'asChangesState') as ChangesState;
|
||||
}
|
||||
|
||||
/** Mint a `SnapshotState`. Callable ONLY from the `Foo/get` response parser in `./jmap.ts`. */
|
||||
export function asSnapshotState(state: unknown): SnapshotState {
|
||||
return certifyStateToken(state, 'asSnapshotState') as SnapshotState;
|
||||
}
|
||||
|
||||
/**
|
||||
* The tag is a REAL, module-private `Symbol()`, deliberately not exported and
|
||||
* deliberately not `declare const ... : unique symbol`.
|
||||
*
|
||||
* - Unexported means no object literal in any other module can produce this
|
||||
* type, so `mintEnumerationCommitment` is the only constructor. Declaring the
|
||||
* interface without a symbol tag would let any module falsify it with a
|
||||
* literal, making the seed path's teeth strictly weaker than
|
||||
* `advanceCursor`'s - which is the path that needs them most.
|
||||
* - `declare const x: unique symbol` is TYPE-LEVEL ONLY and emits no runtime
|
||||
* value, so using it as a computed key throws
|
||||
* `ReferenceError: x is not defined` the first time the mint runs. That
|
||||
* mistake is in the superseded design document; it cost the mobile port a
|
||||
* build failure. A `Symbol()` assigned to a `const` still infers
|
||||
* `unique symbol`, so unforgeability is identical and no cast is needed.
|
||||
*/
|
||||
const enumerationCommitmentTag = Symbol('EnumerationCommitment');
|
||||
|
||||
/**
|
||||
* A durable promise to enumerate. Holding one is what entitles a caller to seed
|
||||
* a cursor from a snapshot state: the snapshot is only a safe cursor because an
|
||||
* enumeration that starts AFTER it is committed to run.
|
||||
*/
|
||||
export interface EnumerationCommitment {
|
||||
readonly [enumerationCommitmentTag]: true;
|
||||
readonly jmapAccountId: string;
|
||||
readonly snapshot: SnapshotState;
|
||||
/** The retention floor the enumeration is working toward. */
|
||||
readonly targetFrom: string;
|
||||
/**
|
||||
* The floor PINNED for this enumeration. Equal to `targetFrom` for a
|
||||
* bootstrap; for a reconcile it is the floor captured at step 0, and the sweep
|
||||
* deletes only against THIS value, never a `targetFrom` that moved while the
|
||||
* reconcile was running.
|
||||
*
|
||||
* Without the pin: widening retention mid-reconcile (very plausible - the
|
||||
* reconcile banner is exactly what prompts someone to go change the setting)
|
||||
* makes the sweep delete against the new wide window while the enumeration
|
||||
* only covered the old narrow one. Everything in the gap is deleted
|
||||
* permanently, because `coveredFrom` is then set to the wider floor and
|
||||
* `/changes` cannot re-deliver pre-existing mail.
|
||||
*/
|
||||
readonly sweepFloor: string;
|
||||
readonly kind: 'bootstrap' | 'reconcile';
|
||||
}
|
||||
|
||||
export function mintEnumerationCommitment(args: {
|
||||
jmapAccountId: string;
|
||||
snapshot: SnapshotState;
|
||||
targetFrom: string;
|
||||
sweepFloor: string;
|
||||
kind: 'bootstrap' | 'reconcile';
|
||||
}): EnumerationCommitment {
|
||||
return {
|
||||
[enumerationCommitmentTag]: true,
|
||||
jmapAccountId: args.jmapAccountId,
|
||||
snapshot: args.snapshot,
|
||||
targetFrom: args.targetFrom,
|
||||
sweepFloor: args.sweepFloor,
|
||||
kind: args.kind,
|
||||
};
|
||||
}
|
||||
|
||||
export function coveragePhaseForCommitment(
|
||||
commitment: EnumerationCommitment,
|
||||
): 'scanning' | 'reconciling' {
|
||||
return commitment.kind === 'bootstrap' ? 'scanning' : 'reconciling';
|
||||
}
|
||||
@@ -0,0 +1,997 @@
|
||||
// The replica store: the encrypted SQLite file, and every read/write against it.
|
||||
//
|
||||
// SYNCHRONOUS ON PURPOSE. `@signalapp/sqlcipher` is a synchronous binding, so
|
||||
// `transaction()` here takes a synchronous callback and no `await` can ever
|
||||
// appear inside a `BEGIN ... COMMIT`. That removes an entire hazard class by
|
||||
// construction: no network call, no timer and no other request can interleave
|
||||
// with a half-applied transaction. Every JMAP fetch happens OUTSIDE a
|
||||
// transaction and the results are applied inside one.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { loadSqlcipher, type SqlcipherDatabase } from '@/lib/mail-index/binding';
|
||||
import { dbSiblings, indexDbPath } from '@/lib/mail-index/paths';
|
||||
import {
|
||||
coverageStateKey, cursorStateKey, FLAGS_KEY, POLICY_KEY, REPLICA_DDL,
|
||||
REPLICA_RECORD_TABLES, REPLICA_SCHEMA_VERSION, REPLICA_TABLES, REPLICA_VERSION_KEY,
|
||||
} from './schema';
|
||||
import {
|
||||
coveragePhaseForCommitment, type ChangesState, type CursorType, type EnumerationCommitment,
|
||||
} from './states';
|
||||
import {
|
||||
defaultFlags, type BodyGiveUpReason, type BodyQueueEntry, type CoverageState,
|
||||
type EnvelopeRow, type FlagsPatch, type MailboxRow, type ReplicaFlags, type SyncCursor,
|
||||
} from './types';
|
||||
|
||||
export class ReplicaUnavailableError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'ReplicaUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface CursorKey {
|
||||
jmapAccountId: string;
|
||||
type: CursorType;
|
||||
}
|
||||
|
||||
/** Retention policy, persisted server-side inside the encrypted store. */
|
||||
export interface RetentionPolicy {
|
||||
envelopeDays: number;
|
||||
bodyDays: number;
|
||||
maxBodyMB: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_POLICY: RetentionPolicy = {
|
||||
// Envelopes are ~1 KB, so a wide window costs kilobytes per message and means
|
||||
// a message never falls out of the offline LIST because of a body size cap.
|
||||
envelopeDays: 180,
|
||||
bodyDays: 30,
|
||||
maxBodyMB: 250,
|
||||
};
|
||||
|
||||
export const POLICY_LIMITS = {
|
||||
envelopeDays: { min: 7, max: 3650 },
|
||||
bodyDays: { min: 1, max: 3650 },
|
||||
maxBodyMB: { min: 16, max: 20_000 },
|
||||
} as const;
|
||||
|
||||
export function clampPolicy(raw: Partial<RetentionPolicy> | null | undefined): RetentionPolicy {
|
||||
const pick = (
|
||||
value: unknown,
|
||||
fallback: number,
|
||||
{ min, max }: { min: number; max: number },
|
||||
): number => {
|
||||
const n = typeof value === 'number' && Number.isFinite(value) ? Math.round(value) : fallback;
|
||||
return Math.min(Math.max(n, min), max);
|
||||
};
|
||||
const envelopeDays = pick(raw?.envelopeDays, DEFAULT_POLICY.envelopeDays, POLICY_LIMITS.envelopeDays);
|
||||
const bodyDays = pick(raw?.bodyDays, DEFAULT_POLICY.bodyDays, POLICY_LIMITS.bodyDays);
|
||||
return {
|
||||
envelopeDays,
|
||||
// The body window can never be wider than the envelope window: a body with
|
||||
// no envelope is an orphan by construction.
|
||||
bodyDays: Math.min(bodyDays, envelopeDays),
|
||||
maxBodyMB: pick(raw?.maxBodyMB, DEFAULT_POLICY.maxBodyMB, POLICY_LIMITS.maxBodyMB),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* `PRAGMA cipher_version` must return a non-empty STRING.
|
||||
*
|
||||
* Checking the row COUNT instead passes vacuously: a non-SQLCipher binding
|
||||
* returns ZERO ROWS for this pragma, and `PRAGMA key = ...` is silently accepted
|
||||
* and does nothing on plain SQLite - no error, a working database, and the mail
|
||||
* sitting on disk in cleartext.
|
||||
*/
|
||||
function assertEncrypted(db: SqlcipherDatabase, dbPath: string): void {
|
||||
const rows = db.pragma('cipher_version');
|
||||
const value =
|
||||
Array.isArray(rows) && rows.length > 0 && rows[0] && typeof rows[0] === 'object'
|
||||
? (rows[0] as Record<string, unknown>).cipher_version
|
||||
: undefined;
|
||||
if (typeof value !== 'string' || value.trim().length === 0) {
|
||||
db.close();
|
||||
throw new ReplicaUnavailableError(
|
||||
`Refusing to use ${path.basename(dbPath)}: the SQLite binding reports no SQLCipher ` +
|
||||
`support (PRAGMA cipher_version returned ${JSON.stringify(rows)}), so the offline ` +
|
||||
`replica would be written in cleartext.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : null;
|
||||
}
|
||||
|
||||
function str(v: unknown): string | null {
|
||||
return typeof v === 'string' ? v : null;
|
||||
}
|
||||
|
||||
export interface OpenReplicaOptions {
|
||||
storeDir: string;
|
||||
accountId: string;
|
||||
/** Raw 32-byte key, from the main process's key service. */
|
||||
key: Buffer;
|
||||
}
|
||||
|
||||
export class ReplicaStore {
|
||||
private constructor(
|
||||
private readonly db: SqlcipherDatabase,
|
||||
readonly dbPath: string,
|
||||
) {}
|
||||
|
||||
static open({ storeDir, accountId, key }: OpenReplicaOptions): ReplicaStore {
|
||||
const Database = loadSqlcipher();
|
||||
if (!Database) {
|
||||
throw new ReplicaUnavailableError(
|
||||
'@signalapp/sqlcipher is not installed for this platform (it is an optional dependency).',
|
||||
);
|
||||
}
|
||||
if (key.length !== 32) {
|
||||
throw new ReplicaUnavailableError(`Replica key must be 32 bytes, got ${key.length}.`);
|
||||
}
|
||||
|
||||
// The SAME file as the search index. See schema.ts for why.
|
||||
const dbPath = indexDbPath(storeDir, accountId);
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: 0o700 });
|
||||
|
||||
const connect = (): SqlcipherDatabase => {
|
||||
const db = new Database(dbPath);
|
||||
// The key pragma must be the FIRST statement on the connection. Hex form
|
||||
// means SQLCipher uses these 32 bytes as the raw key with no KDF.
|
||||
db.pragma(`key = "x'${key.toString('hex')}'"`);
|
||||
assertEncrypted(db, dbPath);
|
||||
return db;
|
||||
};
|
||||
|
||||
let db = connect();
|
||||
let version: number | null;
|
||||
try {
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
// The index and the replica are two connections to one file. WAL lets a
|
||||
// writer and readers coexist, but two WRITERS get SQLITE_BUSY immediately
|
||||
// without this - and both are driven by the same renderer push handler, so
|
||||
// they genuinely do overlap.
|
||||
db.pragma('busy_timeout = 8000');
|
||||
version = readVersion(db);
|
||||
} catch {
|
||||
// A wrong key surfaces here, not at open: SQLCipher reads the header
|
||||
// lazily. The replica is derived data, so there is nothing to recover and
|
||||
// never anything to prompt the user for.
|
||||
db.close();
|
||||
for (const f of dbSiblings(dbPath)) {
|
||||
try { fs.rmSync(f, { force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
db = connect();
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
db.pragma('busy_timeout = 8000');
|
||||
version = null;
|
||||
}
|
||||
|
||||
if (version !== null && version !== REPLICA_SCHEMA_VERSION) version = null;
|
||||
|
||||
if (version === null) {
|
||||
// ALL-OR-NOTHING. Records must never survive while the version row is
|
||||
// gone: a cursor that outlives its records is the one state no amount of
|
||||
// syncing repairs, because `/changes` cannot re-deliver mail that already
|
||||
// existed when the cursor was captured.
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
for (const table of REPLICA_TABLES) db.exec(`DROP TABLE IF EXISTS ${table}`);
|
||||
db.exec(REPLICA_DDL);
|
||||
db.exec('CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL)');
|
||||
db.prepare('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)').run([
|
||||
REPLICA_VERSION_KEY,
|
||||
String(REPLICA_SCHEMA_VERSION),
|
||||
]);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
db.close();
|
||||
throw error;
|
||||
}
|
||||
} else {
|
||||
// The tables exist per the version row, but `CREATE TABLE IF NOT EXISTS`
|
||||
// is cheap and covers a partially-created file from an interrupted open.
|
||||
db.exec(REPLICA_DDL);
|
||||
}
|
||||
|
||||
return new ReplicaStore(db, dbPath);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.db.close(); } catch { /* already closed */ }
|
||||
}
|
||||
|
||||
/**
|
||||
* One SQLite transaction. The callback is SYNCHRONOUS, so nothing can
|
||||
* interleave and no `await` can sit inside `BEGIN ... COMMIT`.
|
||||
*/
|
||||
transaction<T>(fn: () => T): T {
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
const out = fn();
|
||||
this.db.exec('COMMIT');
|
||||
return out;
|
||||
} catch (error) {
|
||||
try { this.db.exec('ROLLBACK'); } catch { /* the commit may have failed */ }
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// ── raw sync_state access ────────────────────────────────────────────────
|
||||
|
||||
private readState<T>(k: string): T | null {
|
||||
const row = this.db.prepare('SELECT v FROM replica_sync_state WHERE k = ?').get([k]);
|
||||
if (!row || typeof row.v !== 'string') return null;
|
||||
try {
|
||||
return JSON.parse(row.v) as T;
|
||||
} catch {
|
||||
// A corrupt state blob is a resync signal, not something to guess at.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private writeState(k: string, value: unknown): void {
|
||||
this.db
|
||||
.prepare('INSERT INTO replica_sync_state (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v')
|
||||
.run([k, JSON.stringify(value)]);
|
||||
}
|
||||
|
||||
// ── policy ───────────────────────────────────────────────────────────────
|
||||
|
||||
getPolicy(): RetentionPolicy {
|
||||
return clampPolicy(this.readState<Partial<RetentionPolicy>>(POLICY_KEY));
|
||||
}
|
||||
|
||||
setPolicy(policy: RetentionPolicy): void {
|
||||
this.writeState(POLICY_KEY, clampPolicy(policy));
|
||||
}
|
||||
|
||||
// ── flags ────────────────────────────────────────────────────────────────
|
||||
|
||||
getFlags(now: number): ReplicaFlags {
|
||||
return this.readState<ReplicaFlags>(FLAGS_KEY) ?? defaultFlags(now);
|
||||
}
|
||||
|
||||
patchFlags(now: number, patch: FlagsPatch): void {
|
||||
const current = this.getFlags(now);
|
||||
this.writeState(FLAGS_KEY, { ...current, ...patch });
|
||||
}
|
||||
|
||||
// ── cursors ──────────────────────────────────────────────────────────────
|
||||
|
||||
getCursor(key: CursorKey): SyncCursor | null {
|
||||
return this.readState<SyncCursor>(cursorStateKey(key.jmapAccountId, key.type));
|
||||
}
|
||||
|
||||
/**
|
||||
* The delta path's ONLY cursor write. The signature is what makes the mobile
|
||||
* client's D4 a compile error here: a `SnapshotState` cannot be passed.
|
||||
*
|
||||
* Throws when the cursor does not exist. A cursor is born from `seedCursor`
|
||||
* and nowhere else; creating one here would be a silent cursor-from-nowhere,
|
||||
* which is the exact class of bug the branded types exist to prevent.
|
||||
*/
|
||||
advanceCursor(key: CursorKey, next: ChangesState): void {
|
||||
const k = cursorStateKey(key.jmapAccountId, key.type);
|
||||
const current = this.readState<SyncCursor>(k);
|
||||
if (!current) {
|
||||
throw new Error(
|
||||
`advanceCursor: no cursor for ${key.type}/${key.jmapAccountId}; seed it first`,
|
||||
);
|
||||
}
|
||||
this.writeState(k, { ...current, state: next, updatedAt: Date.now() } satisfies SyncCursor);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bootstrap / reconcile only. Writes the snapshot state AND the `CoverageState`
|
||||
* it justifies in the SAME transaction, so a seed is never durable without the
|
||||
* durable commitment to enumerate that justifies it.
|
||||
*
|
||||
* Call inside `transaction()`.
|
||||
*/
|
||||
seedCursor(key: CursorKey, commitment: EnumerationCommitment, now: number): void {
|
||||
if (commitment.jmapAccountId !== key.jmapAccountId) {
|
||||
throw new Error('seedCursor: commitment is for a different JMAP account');
|
||||
}
|
||||
const seeded: SyncCursor = {
|
||||
type: key.type,
|
||||
jmapAccountId: key.jmapAccountId,
|
||||
state: commitment.snapshot,
|
||||
drainPending: false,
|
||||
consecutiveFailures: 0,
|
||||
maxChangesRung: 0,
|
||||
updatedAt: now,
|
||||
};
|
||||
this.writeState(cursorStateKey(key.jmapAccountId, key.type), seeded);
|
||||
|
||||
const existing = this.getCoverage(key.jmapAccountId);
|
||||
const next: CoverageState = {
|
||||
jmapAccountId: key.jmapAccountId,
|
||||
// Records stay readable during a reconcile, so what was already covered
|
||||
// stays claimed until the reconcile finishes and sets the pinned floor.
|
||||
coveredFrom: existing?.coveredFrom ?? null,
|
||||
scanCursor: null,
|
||||
targetFrom: commitment.targetFrom,
|
||||
sweepFloor: commitment.sweepFloor,
|
||||
deferredTargetFrom: undefined,
|
||||
gapMarkers: existing?.gapMarkers,
|
||||
phase: coveragePhaseForCommitment(commitment),
|
||||
seen: 0,
|
||||
consecutiveFailures: 0,
|
||||
updatedAt: now,
|
||||
};
|
||||
this.writeState(coverageStateKey(key.jmapAccountId), next);
|
||||
}
|
||||
|
||||
/** Field-level patch. `state` is deliberately NOT patchable - see advance/seed. */
|
||||
patchCursor(
|
||||
key: CursorKey,
|
||||
patch: Partial<Omit<SyncCursor, 'type' | 'jmapAccountId' | 'state'>>,
|
||||
): void {
|
||||
const k = cursorStateKey(key.jmapAccountId, key.type);
|
||||
const current = this.readState<SyncCursor>(k);
|
||||
if (!current) return;
|
||||
this.writeState(k, { ...current, ...patch, updatedAt: Date.now() });
|
||||
}
|
||||
|
||||
// ── coverage ─────────────────────────────────────────────────────────────
|
||||
|
||||
getCoverage(jmapAccountId: string): CoverageState | null {
|
||||
return this.readState<CoverageState>(coverageStateKey(jmapAccountId));
|
||||
}
|
||||
|
||||
patchCoverage(jmapAccountId: string, patch: Partial<CoverageState>): void {
|
||||
const k = coverageStateKey(jmapAccountId);
|
||||
const current = this.readState<CoverageState>(k);
|
||||
if (!current) return;
|
||||
this.writeState(k, { ...current, ...patch, updatedAt: Date.now() });
|
||||
}
|
||||
|
||||
// ── mailboxes ────────────────────────────────────────────────────────────
|
||||
|
||||
upsertMailboxes(rows: readonly MailboxRow[]): number {
|
||||
if (rows.length === 0) return 0;
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO replica_mailbox (jmap_account_id, id, name, parent_id, role, sort_order,
|
||||
total_emails, unread_emails, total_threads, unread_threads, my_rights_json, is_subscribed)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(jmap_account_id, id) DO UPDATE SET
|
||||
name = excluded.name, parent_id = excluded.parent_id, role = excluded.role,
|
||||
sort_order = excluded.sort_order, total_emails = excluded.total_emails,
|
||||
unread_emails = excluded.unread_emails, total_threads = excluded.total_threads,
|
||||
unread_threads = excluded.unread_threads, my_rights_json = excluded.my_rights_json,
|
||||
is_subscribed = excluded.is_subscribed
|
||||
`);
|
||||
for (const r of rows) {
|
||||
stmt.run([
|
||||
r.jmapAccountId, r.id, r.name, r.parentId, r.role, r.sortOrder,
|
||||
r.totalEmails, r.unreadEmails, r.totalThreads, r.unreadThreads,
|
||||
r.myRightsJson, r.isSubscribed ? 1 : 0,
|
||||
]);
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Patches ONLY the four count columns.
|
||||
*
|
||||
* `Mailbox/changes` reports `updatedProperties` as an upper bound of what may
|
||||
* have changed (RFC 8621 s2.2), and counts move on every delivery and every
|
||||
* read. On a busy account this is the difference between patching four
|
||||
* integers and re-fetching every folder object.
|
||||
*/
|
||||
patchMailboxCounts(
|
||||
jmapAccountId: string,
|
||||
id: string,
|
||||
counts: {
|
||||
totalEmails?: number | null; unreadEmails?: number | null;
|
||||
totalThreads?: number | null; unreadThreads?: number | null;
|
||||
},
|
||||
): void {
|
||||
const sets: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
for (const [column, value] of [
|
||||
['total_emails', counts.totalEmails], ['unread_emails', counts.unreadEmails],
|
||||
['total_threads', counts.totalThreads], ['unread_threads', counts.unreadThreads],
|
||||
] as const) {
|
||||
if (value !== undefined) { sets.push(`${column} = ?`); params.push(value); }
|
||||
}
|
||||
if (sets.length === 0) return;
|
||||
this.db
|
||||
.prepare(`UPDATE replica_mailbox SET ${sets.join(', ')} WHERE jmap_account_id = ? AND id = ?`)
|
||||
.run([...params, jmapAccountId, id]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the mailbox row ONLY. Never touches email records.
|
||||
*
|
||||
* Deletion provenance: if the server destroyed the messages too,
|
||||
* `Email/changes` reports them `destroyed`; if it moved them, their
|
||||
* `mailboxIds` update arrives as `updated`. Truth arrives on the Email stream
|
||||
* either way. Inferring deletion from a mailbox disappearing is how a client
|
||||
* loses mail the server still has.
|
||||
*/
|
||||
deleteMailboxes(jmapAccountId: string, ids: readonly string[]): number {
|
||||
if (ids.length === 0) return 0;
|
||||
const stmt = this.db.prepare('DELETE FROM replica_mailbox WHERE jmap_account_id = ? AND id = ?');
|
||||
let n = 0;
|
||||
for (const id of ids) n += stmt.run([jmapAccountId, id]).changes;
|
||||
return n;
|
||||
}
|
||||
|
||||
listMailboxes(jmapAccountId: string): MailboxRow[] {
|
||||
return this.db
|
||||
.prepare('SELECT * FROM replica_mailbox WHERE jmap_account_id = ? ORDER BY sort_order ASC, name ASC')
|
||||
.all([jmapAccountId])
|
||||
.map((r) => ({
|
||||
jmapAccountId: String(r.jmap_account_id),
|
||||
id: String(r.id),
|
||||
name: String(r.name ?? ''),
|
||||
parentId: str(r.parent_id),
|
||||
role: str(r.role),
|
||||
sortOrder: num(r.sort_order),
|
||||
totalEmails: num(r.total_emails),
|
||||
unreadEmails: num(r.unread_emails),
|
||||
totalThreads: num(r.total_threads),
|
||||
unreadThreads: num(r.unread_threads),
|
||||
myRightsJson: str(r.my_rights_json),
|
||||
isSubscribed: r.is_subscribed !== 0,
|
||||
}));
|
||||
}
|
||||
|
||||
// ── envelopes ────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Upserts envelopes and replaces their membership rows.
|
||||
*
|
||||
* `has_body` / `body_bytes` are DELIBERATELY absent from `DO UPDATE SET`: they
|
||||
* belong to the body tier, and resetting them on an idempotent page replay
|
||||
* would look like "body missing" to the backfill job and re-download every
|
||||
* body in the page.
|
||||
*
|
||||
* `cachedAt` is a parameter rather than `Date.now()` because a reconcile must
|
||||
* stamp with its PINNED value - see `CoverageState.reconcileStampedAt`.
|
||||
*/
|
||||
upsertEnvelopes(rows: readonly EnvelopeRow[], cachedAt: number): number {
|
||||
if (rows.length === 0) return 0;
|
||||
const upsert = this.db.prepare(`
|
||||
INSERT INTO replica_envelope (jmap_account_id, id, thread_id, received_at, size, subject,
|
||||
preview, from_json, to_json, cc_json, blob_id, has_attachment, keywords_json,
|
||||
has_body, body_bytes, cached_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?)
|
||||
ON CONFLICT(jmap_account_id, id) DO UPDATE SET
|
||||
thread_id = excluded.thread_id, received_at = excluded.received_at,
|
||||
size = excluded.size, subject = excluded.subject, preview = excluded.preview,
|
||||
from_json = excluded.from_json, to_json = excluded.to_json, cc_json = excluded.cc_json,
|
||||
blob_id = excluded.blob_id, has_attachment = excluded.has_attachment,
|
||||
keywords_json = excluded.keywords_json, cached_at = excluded.cached_at
|
||||
`);
|
||||
const clearMembership = this.db.prepare(
|
||||
'DELETE FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?',
|
||||
);
|
||||
const addMembership = this.db.prepare(
|
||||
'INSERT OR IGNORE INTO replica_email_mailbox (jmap_account_id, email_id, mailbox_id) VALUES (?, ?, ?)',
|
||||
);
|
||||
for (const r of rows) {
|
||||
upsert.run([
|
||||
r.jmapAccountId, r.id, r.threadId, r.receivedAt, r.size, r.subject, r.preview,
|
||||
r.fromJson, r.toJson, r.ccJson, r.blobId, r.hasAttachment ? 1 : 0, r.keywordsJson,
|
||||
cachedAt,
|
||||
]);
|
||||
clearMembership.run([r.jmapAccountId, r.id]);
|
||||
for (const mailboxId of r.mailboxIds) addMembership.run([r.jmapAccountId, r.id, mailboxId]);
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Patches the only two MUTABLE Email properties (RFC 8621 s4.1): `keywords`
|
||||
* and `mailboxIds`. Everything else - body, attachments, headers, receivedAt,
|
||||
* size, threadId, preview, subject, addresses - is immutable for the lifetime
|
||||
* of the id, which is why an `updated` id never needs a body re-fetch.
|
||||
*
|
||||
* No-ops for an id we do not hold, and only touches membership when the
|
||||
* envelope row actually existed, or we would leave membership rows for a
|
||||
* record we do not have.
|
||||
*/
|
||||
patchEnvelopeMutable(
|
||||
jmapAccountId: string,
|
||||
id: string,
|
||||
patch: { keywordsJson: string; mailboxIds: string[] },
|
||||
): boolean {
|
||||
const res = this.db
|
||||
.prepare('UPDATE replica_envelope SET keywords_json = ? WHERE jmap_account_id = ? AND id = ?')
|
||||
.run([patch.keywordsJson, jmapAccountId, id]);
|
||||
if (res.changes === 0) return false;
|
||||
this.db
|
||||
.prepare('DELETE FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?')
|
||||
.run([jmapAccountId, id]);
|
||||
const add = this.db.prepare(
|
||||
'INSERT OR IGNORE INTO replica_email_mailbox (jmap_account_id, email_id, mailbox_id) VALUES (?, ?, ?)',
|
||||
);
|
||||
for (const mailboxId of patch.mailboxIds) add.run([jmapAccountId, id, mailboxId]);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Bulk presence test, so the delta path can filter `updated` ids BEFORE fetching. */
|
||||
whichEnvelopesExist(jmapAccountId: string, ids: readonly string[]): Set<string> {
|
||||
if (ids.length === 0) return new Set();
|
||||
const out = new Set<string>();
|
||||
const stmt = this.db.prepare(
|
||||
'SELECT id FROM replica_envelope WHERE jmap_account_id = ? AND id = ?',
|
||||
);
|
||||
for (const id of ids) {
|
||||
if (stmt.get([jmapAccountId, id])) out.add(id);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Deletes an email everywhere: envelope, body, membership and any queue row. */
|
||||
deleteEmails(jmapAccountId: string, ids: readonly string[]): number {
|
||||
if (ids.length === 0) return 0;
|
||||
const statements = [
|
||||
this.db.prepare('DELETE FROM replica_body WHERE jmap_account_id = ? AND email_id = ?'),
|
||||
this.db.prepare('DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND email_id = ?'),
|
||||
this.db.prepare('DELETE FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?'),
|
||||
];
|
||||
const deleteEnvelope = this.db.prepare(
|
||||
'DELETE FROM replica_envelope WHERE jmap_account_id = ? AND id = ?',
|
||||
);
|
||||
let n = 0;
|
||||
for (const id of ids) {
|
||||
for (const s of statements) s.run([jmapAccountId, id]);
|
||||
n += deleteEnvelope.run([jmapAccountId, id]).changes;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/** Retention eviction: everything strictly older than the floor. */
|
||||
evictEnvelopesBelow(jmapAccountId: string, isoFloor: string): number {
|
||||
const ids = this.db
|
||||
.prepare('SELECT id FROM replica_envelope WHERE jmap_account_id = ? AND received_at < ?')
|
||||
.all([jmapAccountId, isoFloor])
|
||||
.map((r) => String(r.id));
|
||||
return this.deleteEmails(jmapAccountId, ids);
|
||||
}
|
||||
|
||||
/**
|
||||
* The reconcile sweep. Two clauses, and it REFUSES to run without a pinned
|
||||
* stamp rather than deleting unverified records.
|
||||
*/
|
||||
sweep(jmapAccountId: string, sweepFloor: string, reconcileStampedAt: number | undefined): number {
|
||||
if (reconcileStampedAt === undefined) {
|
||||
throw new Error('sweep: no reconcileStampedAt pinned; refusing to delete unverified records');
|
||||
}
|
||||
const notReSeen = this.db
|
||||
.prepare(`
|
||||
SELECT id FROM replica_envelope
|
||||
WHERE jmap_account_id = ? AND received_at >= ? AND cached_at < ?
|
||||
`)
|
||||
.all([jmapAccountId, sweepFloor, reconcileStampedAt])
|
||||
.map((r) => String(r.id));
|
||||
// Records older than the pinned floor cannot be verified by an enumeration
|
||||
// that only covers the window, so they go rather than being kept on faith.
|
||||
// Normally retention has already evicted them.
|
||||
const unverifiable = this.db
|
||||
.prepare('SELECT id FROM replica_envelope WHERE jmap_account_id = ? AND received_at < ?')
|
||||
.all([jmapAccountId, sweepFloor])
|
||||
.map((r) => String(r.id));
|
||||
return this.deleteEmails(jmapAccountId, [...new Set([...notReSeen, ...unverifiable])]);
|
||||
}
|
||||
|
||||
/**
|
||||
* The reconcile stamp must be derived from the DATA, not the clock:
|
||||
* `max(now, maxCachedAt + 1)`. With a frozen or coarse clock,
|
||||
* `cached_at < stamp` matches nothing and the sweep silently deletes nothing.
|
||||
*/
|
||||
maxEnvelopeCachedAt(jmapAccountId: string): number {
|
||||
const row = this.db
|
||||
.prepare('SELECT MAX(cached_at) AS m FROM replica_envelope WHERE jmap_account_id = ?')
|
||||
.get([jmapAccountId]);
|
||||
return num(row?.m) ?? 0;
|
||||
}
|
||||
|
||||
countEnvelopes(jmapAccountId: string): number {
|
||||
const row = this.db
|
||||
.prepare('SELECT COUNT(*) AS n FROM replica_envelope WHERE jmap_account_id = ?')
|
||||
.get([jmapAccountId]);
|
||||
return num(row?.n) ?? 0;
|
||||
}
|
||||
|
||||
/** Envelopes inside the body window with no body yet - the backfill driver. */
|
||||
envelopesWithoutBody(
|
||||
jmapAccountId: string,
|
||||
receivedAfter: string,
|
||||
limit: number,
|
||||
): Array<{ id: string; receivedAt: string; size: number }> {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT id, received_at, size FROM replica_envelope
|
||||
WHERE jmap_account_id = ? AND has_body = 0 AND received_at >= ?
|
||||
ORDER BY received_at DESC LIMIT ?
|
||||
`)
|
||||
.all([jmapAccountId, receivedAfter, limit])
|
||||
.map((r) => ({
|
||||
id: String(r.id),
|
||||
receivedAt: String(r.received_at),
|
||||
size: num(r.size) ?? 0,
|
||||
}));
|
||||
}
|
||||
|
||||
// ── bodies ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Writes a body ONLY if its envelope still exists, and returns whether it did.
|
||||
*
|
||||
* Without the condition, a body fetched moments before its envelope was
|
||||
* destroyed in the same cycle lands as an orphan. This is also exactly why
|
||||
* "run bodies in parallel, it's separate state" is forbidden.
|
||||
*/
|
||||
putBodyIfEnvelopeExists(jmapAccountId: string, emailId: string, json: string): boolean {
|
||||
const envelope = this.db
|
||||
.prepare('SELECT received_at FROM replica_envelope WHERE jmap_account_id = ? AND id = ?')
|
||||
.get([jmapAccountId, emailId]);
|
||||
const receivedAt = str(envelope?.received_at);
|
||||
if (receivedAt === null) return false;
|
||||
const bytes = Buffer.byteLength(json, 'utf8');
|
||||
this.db
|
||||
.prepare(`
|
||||
INSERT INTO replica_body (jmap_account_id, email_id, received_at, json, bytes)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(jmap_account_id, email_id) DO UPDATE SET
|
||||
received_at = excluded.received_at, json = excluded.json, bytes = excluded.bytes
|
||||
`)
|
||||
.run([jmapAccountId, emailId, receivedAt, json, bytes]);
|
||||
this.db
|
||||
.prepare('UPDATE replica_envelope SET has_body = 1, body_bytes = ? WHERE jmap_account_id = ? AND id = ?')
|
||||
.run([bytes, jmapAccountId, emailId]);
|
||||
return true;
|
||||
}
|
||||
|
||||
getBody(jmapAccountId: string, emailId: string): string | null {
|
||||
const row = this.db
|
||||
.prepare('SELECT json FROM replica_body WHERE jmap_account_id = ? AND email_id = ?')
|
||||
.get([jmapAccountId, emailId]);
|
||||
return str(row?.json);
|
||||
}
|
||||
|
||||
deleteBodies(jmapAccountId: string, emailIds: readonly string[]): number {
|
||||
if (emailIds.length === 0) return 0;
|
||||
const del = this.db.prepare(
|
||||
'DELETE FROM replica_body WHERE jmap_account_id = ? AND email_id = ?',
|
||||
);
|
||||
const clearFlag = this.db.prepare(
|
||||
'UPDATE replica_envelope SET has_body = 0, body_bytes = 0 WHERE jmap_account_id = ? AND id = ?',
|
||||
);
|
||||
let n = 0;
|
||||
for (const id of emailIds) {
|
||||
n += del.run([jmapAccountId, id]).changes;
|
||||
clearFlag.run([jmapAccountId, id]);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
bodyBytesTotal(jmapAccountId: string): number {
|
||||
const row = this.db
|
||||
.prepare('SELECT COALESCE(SUM(bytes), 0) AS n FROM replica_body WHERE jmap_account_id = ?')
|
||||
.get([jmapAccountId]);
|
||||
return num(row?.n) ?? 0;
|
||||
}
|
||||
|
||||
countBodies(jmapAccountId: string): number {
|
||||
const row = this.db
|
||||
.prepare('SELECT COUNT(*) AS n FROM replica_body WHERE jmap_account_id = ?')
|
||||
.get([jmapAccountId]);
|
||||
return num(row?.n) ?? 0;
|
||||
}
|
||||
|
||||
/** Oldest bodies first - the cap-eviction order. Envelopes always survive. */
|
||||
oldestBodies(jmapAccountId: string, limit: number): Array<{ emailId: string; bytes: number }> {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT email_id, bytes FROM replica_body WHERE jmap_account_id = ?
|
||||
ORDER BY received_at ASC, email_id ASC LIMIT ?
|
||||
`)
|
||||
.all([jmapAccountId, limit])
|
||||
.map((r) => ({ emailId: String(r.email_id), bytes: num(r.bytes) ?? 0 }));
|
||||
}
|
||||
|
||||
/** Bodies below the body-retention floor. */
|
||||
bodiesBelow(jmapAccountId: string, isoFloor: string, limit: number): string[] {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT email_id FROM replica_body
|
||||
WHERE jmap_account_id = ? AND received_at < ? ORDER BY received_at ASC LIMIT ?
|
||||
`)
|
||||
.all([jmapAccountId, isoFloor, limit])
|
||||
.map((r) => String(r.email_id));
|
||||
}
|
||||
|
||||
/** Bodies whose envelope is gone. Invisible to cap eviction, which walks the body table. */
|
||||
orphanBodies(jmapAccountId: string, limit: number): string[] {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT b.email_id FROM replica_body b
|
||||
LEFT JOIN replica_envelope e
|
||||
ON e.jmap_account_id = b.jmap_account_id AND e.id = b.email_id
|
||||
WHERE b.jmap_account_id = ? AND e.id IS NULL LIMIT ?
|
||||
`)
|
||||
.all([jmapAccountId, limit])
|
||||
.map((r) => String(r.email_id));
|
||||
}
|
||||
|
||||
// ── body queue ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Insert-or-ignore. NEVER resets `attempts` on an existing row, and never
|
||||
* revives a `gave_up` row.
|
||||
*
|
||||
* Returns the number of rows ACTUALLY INSERTED. The distinction matters: the
|
||||
* caller reports this as progress, and reporting attempted-rather-than-inserted
|
||||
* made the mobile engine believe there was unfinished work on every cycle for
|
||||
* as long as any envelope lacked a body - an endless chain of cycles seconds
|
||||
* apart, changing nothing.
|
||||
*/
|
||||
enqueueBodies(entries: readonly BodyQueueEntry[]): number {
|
||||
if (entries.length === 0) return 0;
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT OR IGNORE INTO replica_body_queue
|
||||
(jmap_account_id, email_id, received_at, attempts, next_attempt_at, last_error, gave_up, gave_up_reason)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 0, NULL)
|
||||
`);
|
||||
let inserted = 0;
|
||||
for (const e of entries) {
|
||||
inserted += stmt.run([
|
||||
e.jmapAccountId, e.emailId, e.receivedAt, e.attempts, e.nextAttemptAt ?? null,
|
||||
e.lastError ?? null,
|
||||
]).changes;
|
||||
}
|
||||
return inserted;
|
||||
}
|
||||
|
||||
/** Rows still WANTED: not given up, and past any backoff. Newest first. */
|
||||
takeBodyQueue(jmapAccountId: string, limit: number, now: number): BodyQueueEntry[] {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT * FROM replica_body_queue
|
||||
WHERE jmap_account_id = ? AND gave_up = 0
|
||||
AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
||||
ORDER BY received_at DESC LIMIT ?
|
||||
`)
|
||||
.all([jmapAccountId, now, limit])
|
||||
.map((r) => ({
|
||||
emailId: String(r.email_id),
|
||||
jmapAccountId: String(r.jmap_account_id),
|
||||
receivedAt: String(r.received_at),
|
||||
attempts: num(r.attempts) ?? 0,
|
||||
lastError: str(r.last_error) ?? undefined,
|
||||
nextAttemptAt: num(r.next_attempt_at) ?? undefined,
|
||||
gaveUp: r.gave_up !== 0,
|
||||
gaveUpReason: (str(r.gave_up_reason) as BodyGiveUpReason | null) ?? undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
dequeueBodies(jmapAccountId: string, emailIds: readonly string[]): number {
|
||||
if (emailIds.length === 0) return 0;
|
||||
const stmt = this.db.prepare(
|
||||
'DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND email_id = ?',
|
||||
);
|
||||
let n = 0;
|
||||
for (const id of emailIds) n += stmt.run([jmapAccountId, id]).changes;
|
||||
return n;
|
||||
}
|
||||
|
||||
bumpBodyAttempt(
|
||||
jmapAccountId: string,
|
||||
emailId: string,
|
||||
nextAttemptAt: number,
|
||||
lastError: string,
|
||||
): void {
|
||||
this.db
|
||||
.prepare(`
|
||||
UPDATE replica_body_queue SET attempts = attempts + 1, next_attempt_at = ?, last_error = ?
|
||||
WHERE jmap_account_id = ? AND email_id = ?
|
||||
`)
|
||||
.run([nextAttemptAt, lastError.slice(0, 400), jmapAccountId, emailId]);
|
||||
}
|
||||
|
||||
/** Records a durable terminal state INSTEAD of deleting the row. */
|
||||
markBodyGaveUp(
|
||||
jmapAccountId: string,
|
||||
entries: ReadonlyArray<{ emailId: string; receivedAt: string; reason: BodyGiveUpReason; lastError?: string }>,
|
||||
): void {
|
||||
if (entries.length === 0) return;
|
||||
// A cap-shed body may have no queue row at all (it was fetched and stored
|
||||
// successfully, then evicted), so this must be an upsert rather than an
|
||||
// update - otherwise the mark is silently dropped and the shed/re-download
|
||||
// loop stays open.
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO replica_body_queue
|
||||
(jmap_account_id, email_id, received_at, attempts, next_attempt_at, last_error, gave_up, gave_up_reason)
|
||||
VALUES (?, ?, ?, 0, NULL, ?, 1, ?)
|
||||
ON CONFLICT(jmap_account_id, email_id) DO UPDATE SET
|
||||
gave_up = 1, gave_up_reason = excluded.gave_up_reason,
|
||||
last_error = excluded.last_error, next_attempt_at = NULL
|
||||
`);
|
||||
for (const e of entries) {
|
||||
stmt.run([jmapAccountId, e.emailId, e.receivedAt, e.lastError?.slice(0, 400) ?? null, e.reason]);
|
||||
}
|
||||
}
|
||||
|
||||
listBodyGiveUps(jmapAccountId: string, limit: number): string[] {
|
||||
return this.db
|
||||
.prepare('SELECT email_id FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1 LIMIT ?')
|
||||
.all([jmapAccountId, limit])
|
||||
.map((r) => String(r.email_id));
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETES give-up rows rather than un-flagging them, so a cleared give-up
|
||||
* looks like "never queued" and the backfill pass re-enqueues it with a clean
|
||||
* attempt count.
|
||||
*
|
||||
* Called unconditionally by a completed reconcile: a give-up recorded during
|
||||
* whatever went wrong must not outlive it, or a transient outage would
|
||||
* permanently deny a body with no path back.
|
||||
*/
|
||||
clearBodyGiveUps(jmapAccountId: string, reason?: BodyGiveUpReason): number {
|
||||
if (reason) {
|
||||
return this.db
|
||||
.prepare('DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1 AND gave_up_reason = ?')
|
||||
.run([jmapAccountId, reason]).changes;
|
||||
}
|
||||
return this.db
|
||||
.prepare('DELETE FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1')
|
||||
.run([jmapAccountId]).changes;
|
||||
}
|
||||
|
||||
countWantedBodies(jmapAccountId: string, now: number): number {
|
||||
const row = this.db
|
||||
.prepare(`
|
||||
SELECT COUNT(*) AS n FROM replica_body_queue
|
||||
WHERE jmap_account_id = ? AND gave_up = 0
|
||||
AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
||||
`)
|
||||
.get([jmapAccountId, now]);
|
||||
return num(row?.n) ?? 0;
|
||||
}
|
||||
|
||||
// ── purge ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Wipes records AND the body queue. Leaves `replica_sync_state` (policy, cursors). */
|
||||
clearRecords(): void {
|
||||
for (const table of REPLICA_RECORD_TABLES) this.db.exec(`DELETE FROM ${table}`);
|
||||
}
|
||||
|
||||
/** Everything, cursors included. The only safe pairing with a record wipe. */
|
||||
purgeAll(): void {
|
||||
for (const table of REPLICA_TABLES) this.db.exec(`DELETE FROM ${table}`);
|
||||
}
|
||||
|
||||
// ── read path ────────────────────────────────────────────────────────────
|
||||
|
||||
/** Envelope page for a mailbox, newest first. `mailboxId === null` = all mail. */
|
||||
listEnvelopes(
|
||||
jmapAccountId: string,
|
||||
mailboxId: string | null,
|
||||
limit: number,
|
||||
offset: number,
|
||||
): { rows: Array<Record<string, unknown>>; total: number } {
|
||||
if (mailboxId === null) {
|
||||
const total =
|
||||
num(
|
||||
this.db
|
||||
.prepare('SELECT COUNT(*) AS n FROM replica_envelope WHERE jmap_account_id = ?')
|
||||
.get([jmapAccountId])?.n,
|
||||
) ?? 0;
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT * FROM replica_envelope WHERE jmap_account_id = ?
|
||||
ORDER BY received_at DESC, id DESC LIMIT ? OFFSET ?
|
||||
`)
|
||||
.all([jmapAccountId, limit, offset]);
|
||||
return { rows, total };
|
||||
}
|
||||
const total =
|
||||
num(
|
||||
this.db
|
||||
.prepare('SELECT COUNT(*) AS n FROM replica_email_mailbox WHERE jmap_account_id = ? AND mailbox_id = ?')
|
||||
.get([jmapAccountId, mailboxId])?.n,
|
||||
) ?? 0;
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT e.* FROM replica_envelope e
|
||||
JOIN replica_email_mailbox m
|
||||
ON m.jmap_account_id = e.jmap_account_id AND m.email_id = e.id
|
||||
WHERE e.jmap_account_id = ? AND m.mailbox_id = ?
|
||||
ORDER BY e.received_at DESC, e.id DESC LIMIT ? OFFSET ?
|
||||
`)
|
||||
.all([jmapAccountId, mailboxId, limit, offset]);
|
||||
return { rows, total };
|
||||
}
|
||||
|
||||
getEnvelopeRaw(jmapAccountId: string, id: string): Record<string, unknown> | null {
|
||||
const row = this.db
|
||||
.prepare('SELECT * FROM replica_envelope WHERE jmap_account_id = ? AND id = ?')
|
||||
.get([jmapAccountId, id]);
|
||||
return row ?? null;
|
||||
}
|
||||
|
||||
mailboxIdsFor(jmapAccountId: string, emailId: string): string[] {
|
||||
return this.db
|
||||
.prepare('SELECT mailbox_id FROM replica_email_mailbox WHERE jmap_account_id = ? AND email_id = ?')
|
||||
.all([jmapAccountId, emailId])
|
||||
.map((r) => String(r.mailbox_id));
|
||||
}
|
||||
|
||||
/** Size + freshness, for the Settings surface. */
|
||||
stats(jmapAccountId: string): {
|
||||
mailboxes: number;
|
||||
envelopes: number;
|
||||
bodies: number;
|
||||
bodyBytes: number;
|
||||
wantedBodies: number;
|
||||
giveUps: number;
|
||||
newest: string | null;
|
||||
oldest: string | null;
|
||||
fileBytes: number;
|
||||
} {
|
||||
const mailboxes =
|
||||
num(
|
||||
this.db
|
||||
.prepare('SELECT COUNT(*) AS n FROM replica_mailbox WHERE jmap_account_id = ?')
|
||||
.get([jmapAccountId])?.n,
|
||||
) ?? 0;
|
||||
const range = this.db
|
||||
.prepare('SELECT MIN(received_at) AS lo, MAX(received_at) AS hi FROM replica_envelope WHERE jmap_account_id = ?')
|
||||
.get([jmapAccountId]);
|
||||
let fileBytes = 0;
|
||||
for (const f of dbSiblings(this.dbPath)) {
|
||||
try { fileBytes += fs.statSync(f).size; } catch { /* absent sibling */ }
|
||||
}
|
||||
return {
|
||||
mailboxes,
|
||||
envelopes: this.countEnvelopes(jmapAccountId),
|
||||
bodies: this.countBodies(jmapAccountId),
|
||||
bodyBytes: this.bodyBytesTotal(jmapAccountId),
|
||||
wantedBodies: this.countWantedBodies(jmapAccountId, Date.now()),
|
||||
giveUps:
|
||||
num(
|
||||
this.db
|
||||
.prepare('SELECT COUNT(*) AS n FROM replica_body_queue WHERE jmap_account_id = ? AND gave_up = 1')
|
||||
.get([jmapAccountId])?.n,
|
||||
) ?? 0,
|
||||
newest: str(range?.hi),
|
||||
oldest: str(range?.lo),
|
||||
fileBytes,
|
||||
};
|
||||
}
|
||||
|
||||
/** Every JMAP account id with rows, so the read path can find them without a session. */
|
||||
knownJmapAccountIds(): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const table of ['replica_envelope', 'replica_mailbox'] as const) {
|
||||
for (const r of this.db.prepare(`SELECT DISTINCT jmap_account_id FROM ${table}`).all()) {
|
||||
if (typeof r.jmap_account_id === 'string') ids.add(r.jmap_account_id);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
}
|
||||
|
||||
function readVersion(db: SqlcipherDatabase): number | null {
|
||||
try {
|
||||
const row = db.prepare('SELECT v FROM meta WHERE k = ?').get([REPLICA_VERSION_KEY]);
|
||||
if (!row || row.v === undefined) return null;
|
||||
const n = Number(row.v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
} catch {
|
||||
// `meta` doesn't exist yet - a fresh file.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,169 @@
|
||||
// Persisted shapes for the offline replica.
|
||||
//
|
||||
// Note that `SyncCursor.state` is declared as a plain `string` here while
|
||||
// `./states.ts` goes to some trouble to brand it. That is deliberate: a token
|
||||
// that has round-tripped through JSON has no provenance left to certify. The
|
||||
// brands guard the WRITE PATHS (`advanceCursor` / `seedCursor`), which is where
|
||||
// provenance is actually decided; the row is just a row.
|
||||
|
||||
import type { CursorType } from './states';
|
||||
|
||||
export interface SyncCursor {
|
||||
type: CursorType;
|
||||
jmapAccountId: string;
|
||||
/** A ChangesState from this (type, jmapAccountId), or a seeded SnapshotState. */
|
||||
state: string;
|
||||
/** True when the last page reported `hasMoreChanges` - a drain is unfinished. */
|
||||
drainPending: boolean;
|
||||
/** Set when the server invalidated us. Cleared only by a COMPLETED reconcile. */
|
||||
invalidatedAt?: number;
|
||||
invalidatedReason?: 'cannotCalculateChanges' | 'oldStateMismatch' | 'corruptState' | 'manual';
|
||||
/**
|
||||
* Anti-wedge counters, PER CURSOR rather than per account. With the counters
|
||||
* shared, a healthy Mailbox cursor resetting them every cycle meant a failing
|
||||
* Email cursor never escalated and never advanced again - silently, forever.
|
||||
*/
|
||||
consecutiveFailures: number;
|
||||
/** The `sinceState` that failed; escalation only counts failures at the same position. */
|
||||
lastFailedState?: string;
|
||||
/** Current rung of the `maxChanges` ladder. */
|
||||
maxChangesRung: 0 | 1 | 2 | 3;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
export interface CoverageState {
|
||||
jmapAccountId: string;
|
||||
/** ISO. Oldest `receivedAt` for which the ENVELOPE tier is known-complete. */
|
||||
coveredFrom: string | null;
|
||||
/** ISO. Ascending scan resume point; null when not scanning. */
|
||||
scanCursor: string | null;
|
||||
/** The retention floor this scan is working toward. */
|
||||
targetFrom: string;
|
||||
/** The floor PINNED at reconcile start. The sweep deletes only against this. */
|
||||
sweepFloor?: string;
|
||||
/** Set when a retention change arrived mid-reconcile; applied after the sweep. */
|
||||
deferredTargetFrom?: string;
|
||||
/**
|
||||
* The `cached_at` stamp a reconcile's enumeration writes onto every envelope it
|
||||
* re-sees, pinned when the reconcile starts.
|
||||
*
|
||||
* This is a multi-cycle, crash-resumable "seen set" implemented as ONE
|
||||
* INTEGER, with no seen-ids table: the enumeration re-upserts each surviving
|
||||
* envelope, refreshing its `cached_at`, so the sweep is
|
||||
* `received_at >= sweepFloor AND cached_at < reconcileStampedAt`. A record the
|
||||
* enumeration never reached keeps its older stamp and is swept; a record the
|
||||
* LIVE delta path creates mid-reconcile gets `Date.now() >= stamp` and
|
||||
* survives, which is exactly right.
|
||||
*
|
||||
* The stamp must be derived from the DATA, not the clock:
|
||||
* `max(now, maxCachedAt + 1)`. With a frozen or coarse clock, `cached_at <
|
||||
* stamp` matches nothing and the sweep deletes nothing.
|
||||
*/
|
||||
reconcileStampedAt?: number;
|
||||
/** Durable trace of any tie-cluster skip taken by the last-resort paging rung. */
|
||||
gapMarkers?: Array<{ from: string; to: string; reason: 'tie-cluster-skip'; at: number }>;
|
||||
phase: 'never-run' | 'scanning' | 'reconciling' | 'complete';
|
||||
/** Progress, for the UI only. Never load-bearing. */
|
||||
seen: number;
|
||||
consecutiveFailures: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
export interface BodyQueueEntry {
|
||||
emailId: string;
|
||||
jmapAccountId: string;
|
||||
/** Drives priority: newest first. */
|
||||
receivedAt: string;
|
||||
/** NEVER reset by a re-enqueue. */
|
||||
attempts: number;
|
||||
lastError?: string;
|
||||
nextAttemptAt?: number;
|
||||
/**
|
||||
* Durable terminal state. A gave-up row is KEPT rather than deleted, precisely
|
||||
* so the backfill job cannot resurrect it - its driver is "envelope without a
|
||||
* body", which by itself cannot distinguish "not fetched yet" from
|
||||
* "deliberately not kept". Cleared wholesale by a completed reconcile, so a
|
||||
* transient outage self-heals.
|
||||
*/
|
||||
gaveUp?: boolean;
|
||||
gaveUpReason?: 'attempts' | 'notFound' | 'shed-by-cap';
|
||||
}
|
||||
|
||||
export type BodyGiveUpReason = NonNullable<BodyQueueEntry['gaveUpReason']>;
|
||||
|
||||
/** Per-account flags. A VIEW over field-level patches, never written whole. */
|
||||
export interface ReplicaFlags {
|
||||
/** Sticky until a reconcile completes. Survives restarts. */
|
||||
resyncRequired: boolean;
|
||||
/** Rolling count + window start for the reconcile ceiling. */
|
||||
reconcilesInWindow: number;
|
||||
reconcileWindowStartedAt: number;
|
||||
/**
|
||||
* Last observed retention floor, for the clock-jump guard.
|
||||
*
|
||||
* This must hold the floor that was actually USED, never the suspicious one
|
||||
* that was rejected - see `retention.ts` for the wipe that the other choice
|
||||
* caused.
|
||||
*/
|
||||
lastWindowFloor?: string;
|
||||
/**
|
||||
* The `envelopeDays` that produced `lastWindowFloor`.
|
||||
*
|
||||
* The computed floor moves for TWO independent reasons - the clock changing
|
||||
* and the SETTING changing - and guarding a setting change is wrong: it is
|
||||
* explicit user intent, not a glitch. Recording the policy alongside the floor
|
||||
* is what tells them apart.
|
||||
*/
|
||||
lastEnvelopeDays?: number;
|
||||
/**
|
||||
* The body-tier byte cap in force last cycle. A RAISE must revive bodies
|
||||
* previously shed for space, which is otherwise a durable refusal.
|
||||
*/
|
||||
lastMaxBodyBytes?: number;
|
||||
lastCycleAt?: number;
|
||||
lastCycleOk?: boolean;
|
||||
lastCycleError?: string;
|
||||
}
|
||||
|
||||
export function defaultFlags(now: number): ReplicaFlags {
|
||||
return {
|
||||
resyncRequired: false,
|
||||
reconcilesInWindow: 0,
|
||||
reconcileWindowStartedAt: now,
|
||||
};
|
||||
}
|
||||
|
||||
export type FlagsPatch = Partial<ReplicaFlags>;
|
||||
|
||||
/** The envelope tier, as stored. */
|
||||
export interface EnvelopeRow {
|
||||
jmapAccountId: string;
|
||||
id: string;
|
||||
threadId: string | null;
|
||||
receivedAt: string;
|
||||
size: number | null;
|
||||
subject: string | null;
|
||||
preview: string | null;
|
||||
fromJson: string | null;
|
||||
toJson: string | null;
|
||||
ccJson: string | null;
|
||||
blobId: string | null;
|
||||
hasAttachment: boolean;
|
||||
keywordsJson: string;
|
||||
mailboxIds: string[];
|
||||
}
|
||||
|
||||
export interface MailboxRow {
|
||||
jmapAccountId: string;
|
||||
id: string;
|
||||
name: string;
|
||||
parentId: string | null;
|
||||
role: string | null;
|
||||
sortOrder: number | null;
|
||||
totalEmails: number | null;
|
||||
unreadEmails: number | null;
|
||||
totalThreads: number | null;
|
||||
unreadThreads: number | null;
|
||||
myRightsJson: string | null;
|
||||
isSubscribed: boolean;
|
||||
}
|
||||
Reference in New Issue
Block a user