Merge branch 'claude/electron-offline-design' into dev
Encrypted SQLite/FTS5 offline search index for the Electron desktop client: event-driven reindex (mail, calendar, contacts, files) driven off the existing JMAP push connection, per-account keys held in OS keychain via safeStorage, search API returns ranked context ready for an LLM/RAG prompt.
This commit is contained in:
Bernd Rodler
@@ -75,6 +75,7 @@ export class DemoJMAPClient implements IJMAPClient {
|
||||
getMaxDelayedSend(): number { return 30 * 24 * 60 * 60; }
|
||||
hasDelayedSend(): boolean { return true; }
|
||||
getEventSourceUrl(): string | null { return null; }
|
||||
getWebSocketUrl(): string | null { return null; }
|
||||
supportsEmailSubmission(): boolean { return true; }
|
||||
supportsQuota(): boolean { return true; }
|
||||
supportsVacationResponse(): boolean { return true; }
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Detects whether the app is running inside the VNCmail+ (Bulwark) Electron
|
||||
// desktop shell and wraps the native notification bridge that
|
||||
// electron/preload.ts exposes via contextBridge. Mirrors how lib/web-push.ts
|
||||
// mirrors the React Native push flow - same idea, different native API:
|
||||
// PushManager/service-worker there, Electron's own Notification API here.
|
||||
//
|
||||
// Web/PWA deployments never get `window.vnc` at all (contextBridge only
|
||||
// exists inside the Electron shell), so `isElectronShell()` is false there
|
||||
// and callers should keep using the lib/web-push.ts + public/sw.js path.
|
||||
// Wiring this bridge up to real mail-delivery events (JMAP WebSocket push
|
||||
// vs. polling) is a separate, later decision - this module is only the
|
||||
// plumbing.
|
||||
|
||||
export interface ShowNotificationOptions {
|
||||
body?: string;
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
export interface ShowNotificationResult {
|
||||
shown: boolean;
|
||||
}
|
||||
|
||||
export interface VncElectronBridge {
|
||||
isElectron: true;
|
||||
showNotification: (
|
||||
title: string,
|
||||
options?: ShowNotificationOptions,
|
||||
) => Promise<ShowNotificationResult>;
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
vnc?: VncElectronBridge;
|
||||
}
|
||||
}
|
||||
|
||||
export function isElectronShell(): boolean {
|
||||
return typeof window !== "undefined" && window.vnc?.isElectron === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shows a notification via Electron's native Notification API when running
|
||||
* inside the desktop shell. Resolves to false (never throws) when not
|
||||
* running in Electron, or when the main process reports notifications
|
||||
* unsupported on this OS/session - callers can fall back to the
|
||||
* service-worker push path (lib/web-push.ts) in that case.
|
||||
*/
|
||||
export async function showElectronNotification(
|
||||
title: string,
|
||||
options?: ShowNotificationOptions,
|
||||
): Promise<boolean> {
|
||||
if (!isElectronShell()) return false;
|
||||
const result = await window.vnc!.showNotification(title, options);
|
||||
return result.shown;
|
||||
}
|
||||
@@ -35,6 +35,7 @@ export interface IJMAPClient {
|
||||
getMaxDelayedSend(accountId?: string): number;
|
||||
hasDelayedSend(accountId?: string): boolean;
|
||||
getEventSourceUrl(): string | null;
|
||||
getWebSocketUrl(): string | null;
|
||||
supportsEmailSubmission(): boolean;
|
||||
supportsQuota(): boolean;
|
||||
supportsVacationResponse(): boolean;
|
||||
|
||||
+440
-7
@@ -953,6 +953,36 @@ export class JMAPClient implements IJMAPClient {
|
||||
if (session.eventSourceUrl) {
|
||||
session.eventSourceUrl = this.rewriteSessionUrl(session.eventSourceUrl);
|
||||
}
|
||||
const wsCapability = session.capabilities?.["urn:ietf:params:jmap:websocket"] as
|
||||
| { url?: string }
|
||||
| undefined;
|
||||
if (wsCapability?.url) {
|
||||
wsCapability.url = this.rewriteWebSocketUrl(wsCapability.url);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Same reasoning as rewriteSessionUrl (a reverse proxy may advertise its
|
||||
* own internal hostname), but scheme-aware: unlike apiUrl/eventSourceUrl,
|
||||
* this URL is never touched by fetch() - it goes straight into `new
|
||||
* WebSocket(...)`, and a ws/wss URL can never share an origin string with
|
||||
* an http/https serverUrl even when the host is identical, so reusing
|
||||
* rewriteSessionUrl's plain origin-equality check would rewrite EVERY
|
||||
* websocket URL onto an http(s) scheme and break the constructor outright.
|
||||
*/
|
||||
private rewriteWebSocketUrl(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const server = new URL(this.serverUrl);
|
||||
const expectedScheme = server.protocol === "https:" ? "wss:" : "ws:";
|
||||
if (parsed.host === server.host && parsed.protocol === expectedScheme) {
|
||||
return url;
|
||||
}
|
||||
const pathAndRest = url.slice(url.indexOf("/", url.indexOf("//") + 2));
|
||||
return `${expectedScheme}//${server.host}${pathAndRest}`;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
private async request(methodCalls: JMAPMethodCall[], using?: string[]): Promise<JMAPResponse> {
|
||||
@@ -3761,6 +3791,22 @@ export class JMAPClient implements IJMAPClient {
|
||||
return this.session.eventSourceUrl || coreCapability?.eventSourceUrl || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* RFC 8887 (JMAP over WebSocket) push endpoint, advertised under the
|
||||
* `urn:ietf:params:jmap:websocket` capability (not a root session field
|
||||
* like eventSourceUrl - it's nested the same way every other JMAP
|
||||
* extension capability is). Rewritten to the client's own server host in
|
||||
* rewriteSessionUrls() at connect time, same reasoning as apiUrl/
|
||||
* downloadUrl/eventSourceUrl. Returns null for servers that don't
|
||||
* advertise it - callers fall back to SSE/polling.
|
||||
*/
|
||||
getWebSocketUrl(): string | null {
|
||||
const wsCapability = this.capabilities["urn:ietf:params:jmap:websocket"] as
|
||||
| { url?: string; supportsPush?: boolean }
|
||||
| undefined;
|
||||
return wsCapability?.url || null;
|
||||
}
|
||||
|
||||
getAccountId(): string {
|
||||
return this.accountId;
|
||||
}
|
||||
@@ -5982,12 +6028,51 @@ export class JMAPClient implements IJMAPClient {
|
||||
private visibilityHandler: (() => void) | null = null;
|
||||
private onlineHandler: (() => void) | null = null;
|
||||
|
||||
// JMAP-over-WebSocket (RFC 8887) push - preferred over SSE when the server
|
||||
// advertises it (getWebSocketUrl()), since it's the transport the desktop
|
||||
// shell's main process eventually wants for background/no-window
|
||||
// notifications (see electron/preload.ts's showNotification bridge).
|
||||
// Falls back to the existing SSE/polling chain below when unsupported OR
|
||||
// when the handshake itself keeps failing (see wsPermanentlyDisabled).
|
||||
//
|
||||
// KNOWN LIMITATION, confirmed empirically against the sandbox server this
|
||||
// was built against (stalwart.sandbox.vnc.de): its /jmap/ws endpoint
|
||||
// requires the same HTTP Basic/Bearer Authorization header as every other
|
||||
// JMAP endpoint on the WebSocket UPGRADE request itself (curling it with
|
||||
// no Authorization header returns a plain 401 before any WS frame is
|
||||
// possible). The browser WebSocket constructor has no way to attach
|
||||
// custom headers to that handshake (a WHATWG spec restriction, not an
|
||||
// Electron/browser quirk - credentials in the URL are actively rejected
|
||||
// too), so from this renderer-side client there is no way to satisfy that
|
||||
// auth requirement. Against a server with this exact auth model, every
|
||||
// connection attempt below will fail at the handshake and the circuit
|
||||
// breaker (wsPermanentlyDisabled) will fall back to SSE after a few quick
|
||||
// retries - which is not a bug in this code, it is what actually happens
|
||||
// on the wire. It's still implemented for real (not stubbed) because (a)
|
||||
// it's fully spec-correct and will light up automatically against any
|
||||
// server whose WS endpoint doesn't have this requirement - e.g. one
|
||||
// sitting behind a proxy that authenticates via cookies instead - with no
|
||||
// further changes, and (b) the alternative (opening it from Electron's
|
||||
// main process via a header-capable client like the `ws` package) would
|
||||
// mean piping raw credentials from the renderer to the main process over
|
||||
// IPC, which is a materially bigger security-sensitive change than what
|
||||
// was scoped here.
|
||||
private ws: WebSocket | null = null;
|
||||
private wsReconnectTimeout: NodeJS.Timeout | null = null;
|
||||
private wsReconnectAttempts: number = 0;
|
||||
private wsConsecutiveFailures: number = 0;
|
||||
private wsPermanentlyDisabled: boolean = false;
|
||||
private wsHeartbeatTimer: NodeJS.Timeout | null = null;
|
||||
private lastWSActivity: number = 0;
|
||||
|
||||
private static readonly STATE_TYPE_MAP: Record<string, string> = {
|
||||
'Mailbox/get': 'Mailbox',
|
||||
'Email/get': 'Email',
|
||||
'Calendar/get': 'Calendar',
|
||||
'CalendarEvent/get': 'CalendarEvent',
|
||||
'SieveScript/get': 'SieveScript',
|
||||
'ContactCard/get': 'ContactCard',
|
||||
'FileNode/get': 'FileNode',
|
||||
};
|
||||
|
||||
private static readonly POLLING_INTERVAL = 3_000;
|
||||
@@ -5998,20 +6083,315 @@ export class JMAPClient implements IJMAPClient {
|
||||
private static readonly SSE_RECONNECT_DELAY = 3_000;
|
||||
private static readonly SSE_PING_TIMEOUT = 90_000; // 3x the 30s ping interval
|
||||
|
||||
// Exponential backoff with full jitter (0..cap), doubling from a 200ms
|
||||
// base and capping at 5s.
|
||||
//
|
||||
// Deliberately much tighter than a "normal" reconnect ladder (something
|
||||
// like 1s/30s would be the textbook default for a flaky network) - and
|
||||
// tuned from a real, measured failure mode, not guessed: the auth
|
||||
// limitation described above fails FAST and DETERMINISTICALLY (the
|
||||
// handshake is rejected before the socket ever opens, in well under a
|
||||
// second, every single time), not slowly. Verified empirically (see
|
||||
// integration/tests/11-electron-notification.spec.ts's development) that
|
||||
// the original 1s-base/30s-cap/5-attempt ladder let the circuit breaker
|
||||
// take up to ~31s to trip, during which there is NO live push at all
|
||||
// (WS hasn't succeeded and hasn't given up yet, so SSE never even starts
|
||||
// connecting) - a real mail delivery landing in that window was missed
|
||||
// entirely, since SSE only streams future changes and does no catch-up
|
||||
// fetch on connect. This tighter ladder closes that gap to a fraction of
|
||||
// a second for the fast-fail case while remaining exactly as protective
|
||||
// for a genuinely slow/flaky network: a hanging attempt is still bounded
|
||||
// by the browser's own WebSocket connect timeout regardless of these
|
||||
// constants, which govern only the GAP between attempts, not how long a
|
||||
// single attempt is allowed to hang.
|
||||
private static readonly WS_RECONNECT_BASE_DELAY = 200;
|
||||
private static readonly WS_RECONNECT_MAX_DELAY = 5_000;
|
||||
// App-level heartbeat: a WebSocket can sit in "open" readyState for a long
|
||||
// time after the underlying network path is actually gone (sleep, network
|
||||
// switch, a NAT/proxy that silently drops idle connections) - TCP alone
|
||||
// won't always surface that promptly. Send a lightweight JMAP request
|
||||
// every 30s and force-reconnect if nothing (heartbeat response OR a real
|
||||
// push) has arrived within 3x that window, mirroring the SSE ping monitor
|
||||
// above.
|
||||
private static readonly WS_HEARTBEAT_INTERVAL = 30_000;
|
||||
private static readonly WS_ACTIVITY_TIMEOUT = 90_000;
|
||||
// Give up on WS for this client instance after this many CONSECUTIVE
|
||||
// attempts that never reach "open" (a connection that opened fine and
|
||||
// later dropped does not count - see connectWebSocket's openedSuccessfully
|
||||
// tracking). Bounds the cost of the auth limitation described above to a
|
||||
// handful of quick handshake attempts (with the tightened backoff above,
|
||||
// well under a second in the common fast-fail case) instead of retrying a
|
||||
// request that can never succeed, forever, for the lifetime of the session.
|
||||
private static readonly WS_MAX_CONSECUTIVE_FAILURES = 3;
|
||||
|
||||
/** getWebSocketUrl(), gated by the circuit breaker above. */
|
||||
private effectiveWebSocketUrl(): string | null {
|
||||
return this.wsPermanentlyDisabled ? null : this.getWebSocketUrl();
|
||||
}
|
||||
|
||||
setupPushNotifications(): boolean {
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
// SSE covers the primary account only; keep shared accounts fresh too.
|
||||
const wsUrl = this.effectiveWebSocketUrl();
|
||||
if (wsUrl) {
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.connectWebSocket(wsUrl);
|
||||
// Prime the polling baseline (pollingStates) in parallel with the WS
|
||||
// attempt, not just for shared/secondary accounts below - if WS ends
|
||||
// up failing and falling back (fallbackFromWebSocket()), this is what
|
||||
// lets that fallback reconcile anything that changed to the PRIMARY
|
||||
// account while WS was still churning through retries. Without an
|
||||
// early baseline, a change in that window would be silently missed
|
||||
// entirely: SSE only streams changes from the moment it connects
|
||||
// onward (no catch-up on connect), so the one thing that CAN catch up
|
||||
// is a diff against a state snapshot taken before the gap started.
|
||||
void this.fetchCurrentStates();
|
||||
// Not confirmed either way whether this server's WebSocket push fans
|
||||
// out to shared/secondary accounts or, like Stalwart's SSE, covers the
|
||||
// primary account only - keep the same secondary poll running under
|
||||
// WS that SSE already needed, rather than assume broader coverage and
|
||||
// risk shared-account counters going stale.
|
||||
this.startSecondaryAccountPoll();
|
||||
} else {
|
||||
// The fallback poll already covers every session account.
|
||||
this.startPollingFallback();
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
// SSE covers the primary account only; keep shared accounts fresh too.
|
||||
this.startSecondaryAccountPoll();
|
||||
} else {
|
||||
// The fallback poll already covers every session account.
|
||||
this.startPollingFallback();
|
||||
}
|
||||
}
|
||||
this.setupBrowserEventListeners();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the RFC 8887 JMAP-over-WebSocket connection and subscribes to
|
||||
* push for every data type (`WebSocketPushEnable` with dataTypes: null).
|
||||
* Reconnect on close/error is handled by scheduleWSReconnect() below with
|
||||
* exponential backoff - this method only ever represents a single
|
||||
* connection attempt.
|
||||
*/
|
||||
private connectWebSocket(wsUrl: string): void {
|
||||
if (this.isRateLimited()) {
|
||||
this.scheduleWSReconnect();
|
||||
return;
|
||||
}
|
||||
|
||||
let socket: WebSocket;
|
||||
try {
|
||||
socket = new WebSocket(wsUrl, "jmap");
|
||||
} catch {
|
||||
// New URL()-level failures (malformed URL) - retry later in case a
|
||||
// session refresh fixes it; getWebSocketUrl() re-reads capabilities
|
||||
// fresh on every attempt.
|
||||
this.scheduleWSReconnect();
|
||||
return;
|
||||
}
|
||||
|
||||
this.ws = socket;
|
||||
const isCurrent = () => this.ws === socket;
|
||||
// Tracks whether THIS specific attempt ever reached "open" - a socket
|
||||
// that opened fine and dropped later (real network blip on an
|
||||
// established connection) must not count toward the circuit breaker the
|
||||
// same way a handshake that never completes does (see
|
||||
// wsPermanentlyDisabled's declaration above for why the latter needs
|
||||
// one at all).
|
||||
let openedSuccessfully = false;
|
||||
|
||||
socket.addEventListener("open", () => {
|
||||
if (!isCurrent()) return;
|
||||
openedSuccessfully = true;
|
||||
// A real connection succeeded - both counters reset: the backoff
|
||||
// ladder no longer applies to whatever eventually causes the NEXT
|
||||
// disconnect, and the "give up on WS entirely" counter only tracks
|
||||
// CONSECUTIVE handshake failures.
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.wsConsecutiveFailures = 0;
|
||||
this.lastWSActivity = Date.now();
|
||||
this.startWSHeartbeat(socket);
|
||||
try {
|
||||
socket.send(JSON.stringify({ "@type": "WebSocketPushEnable", dataTypes: null }));
|
||||
} catch {
|
||||
// send() can throw if the socket already closed between "open"
|
||||
// firing and this line running - the "close" handler below will
|
||||
// schedule a reconnect regardless.
|
||||
}
|
||||
});
|
||||
|
||||
socket.addEventListener("message", (event) => {
|
||||
if (!isCurrent()) return;
|
||||
this.lastWSActivity = Date.now();
|
||||
this.processWebSocketMessage(typeof event.data === "string" ? event.data : "");
|
||||
});
|
||||
|
||||
socket.addEventListener("close", () => {
|
||||
if (!isCurrent()) return;
|
||||
this.stopWSHeartbeat();
|
||||
this.ws = null;
|
||||
if (this.intentionallyDisconnected) return;
|
||||
|
||||
if (!openedSuccessfully) {
|
||||
this.wsConsecutiveFailures += 1;
|
||||
if (this.wsConsecutiveFailures >= JMAPClient.WS_MAX_CONSECUTIVE_FAILURES) {
|
||||
// The handshake itself is what's failing, repeatedly - most
|
||||
// commonly (confirmed against this client's own reference
|
||||
// server) because the WS endpoint requires an Authorization
|
||||
// header the browser WebSocket API cannot attach. Retrying that
|
||||
// forever would just hammer the server every ~30s with a request
|
||||
// that can never succeed from here. Give up on WS for the rest of
|
||||
// this client instance's life and stay on SSE/polling, which
|
||||
// don't have this limitation.
|
||||
this.wsPermanentlyDisabled = true;
|
||||
console.warn(
|
||||
'[JMAP] WebSocket push failed to establish after repeated attempts; falling back to SSE/polling for this session.',
|
||||
);
|
||||
this.fallbackFromWebSocket();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
this.scheduleWSReconnect();
|
||||
});
|
||||
|
||||
// WebSocket always fires "close" right after "error" - the reconnect
|
||||
// logic lives entirely in the "close" handler above so there is exactly
|
||||
// one path that schedules a retry, not two racing each other.
|
||||
}
|
||||
|
||||
/** Whatever push transport SSE would have used, now that WS has given up. */
|
||||
private fallbackFromWebSocket(): void {
|
||||
void this.reconcileAfterWebSocketFallback();
|
||||
}
|
||||
|
||||
/**
|
||||
* Diffs against the baseline setupPushNotifications() primed via
|
||||
* fetchCurrentStates() when the WS attempt began - BEFORE either branch
|
||||
* below gets a chance to erase that opportunity (startPollingFallback()
|
||||
* unconditionally overwrites the same baseline via its own
|
||||
* fetchCurrentStates() call; connectSSE() only ever streams changes from
|
||||
* the moment it connects onward, no catch-up). This is what catches a
|
||||
* real mail delivery (or any other tracked change) that happened to the
|
||||
* primary account while WS was still churning through retries, which
|
||||
* neither of those two paths would otherwise ever notice - confirmed as a
|
||||
* real, not theoretical, gap during this feature's own development (see
|
||||
* the WS_RECONNECT_BASE_DELAY comment above).
|
||||
*
|
||||
* Not airtight: if the early fetchCurrentStates() from
|
||||
* setupPushNotifications() hasn't itself completed yet by the time this
|
||||
* runs, there's nothing to diff against and this call just establishes
|
||||
* the baseline instead of detecting drift. In practice that race needs a
|
||||
* pathologically slow state-fetch racing an unusually fast WS failure,
|
||||
* and the tightened backoff above (worst case ~1.75s to exhaust 3
|
||||
* attempts) gives that fetch a lot more room to finish first than the
|
||||
* original 31s-worst-case ladder did.
|
||||
*/
|
||||
private async reconcileAfterWebSocketFallback(): Promise<void> {
|
||||
await this.checkForStateChanges();
|
||||
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
this.startSecondaryAccountPoll();
|
||||
} else {
|
||||
this.startPollingFallback();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses one WebSocket text frame. Per RFC 8887 the server can send
|
||||
* Response, StateChange, or PushState frames; only StateChange is
|
||||
* consumed today (method calls aren't yet routed over this socket -
|
||||
* request()/authenticatedFetch() still uses plain HTTP), so anything else
|
||||
* is silently ignored rather than treated as an error.
|
||||
*/
|
||||
private processWebSocketMessage(raw: string): void {
|
||||
if (!raw) return;
|
||||
let message: { "@type"?: string; changed?: StateChange["changed"] } | null = null;
|
||||
try {
|
||||
message = JSON.parse(raw);
|
||||
} catch {
|
||||
return; // malformed frame - ignore, matches processSSEEvent's handling
|
||||
}
|
||||
if (message?.["@type"] === "StateChange" && message.changed) {
|
||||
this.stateChangeCallback?.({ "@type": "StateChange", changed: message.changed });
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleWSReconnect(): void {
|
||||
if (this.intentionallyDisconnected) return;
|
||||
if (this.wsReconnectTimeout) return; // already scheduled - don't stack retries
|
||||
|
||||
const wsUrl = this.effectiveWebSocketUrl();
|
||||
if (!wsUrl) {
|
||||
// Either the server capability disappeared (e.g. a session refresh
|
||||
// dropped WebSocket support) or the circuit breaker already tripped -
|
||||
// fall back to whatever push transport is still available instead of
|
||||
// retrying a URL that's gone or a handshake that won't succeed.
|
||||
this.fallbackFromWebSocket();
|
||||
return;
|
||||
}
|
||||
|
||||
const attempt = this.wsReconnectAttempts;
|
||||
this.wsReconnectAttempts += 1;
|
||||
const exponential = JMAPClient.WS_RECONNECT_BASE_DELAY * Math.pow(2, attempt);
|
||||
const cap = Math.min(exponential, JMAPClient.WS_RECONNECT_MAX_DELAY);
|
||||
// Full jitter (uniform 0..cap) rather than a fixed exponential delay -
|
||||
// spreads reconnect attempts out after a shared network blip (proxy
|
||||
// restart, wifi handoff affecting every open tab/window at once)
|
||||
// instead of having them all retry in lockstep.
|
||||
const delay = Math.random() * cap;
|
||||
|
||||
this.wsReconnectTimeout = setTimeout(() => {
|
||||
this.wsReconnectTimeout = null;
|
||||
if (this.isRateLimited()) {
|
||||
this.scheduleWSReconnect();
|
||||
return;
|
||||
}
|
||||
this.connectWebSocket(wsUrl);
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private startWSHeartbeat(socket: WebSocket): void {
|
||||
this.stopWSHeartbeat();
|
||||
this.wsHeartbeatTimer = setInterval(() => {
|
||||
if (this.ws !== socket) return;
|
||||
if (Date.now() - this.lastWSActivity > JMAPClient.WS_ACTIVITY_TIMEOUT) {
|
||||
// Silently dead connection (sleep/network switch/idle proxy) - the
|
||||
// socket can still report readyState OPEN long after the underlying
|
||||
// path is gone. Force-close; the "close" handler schedules the
|
||||
// reconnect via the normal backoff path.
|
||||
this.stopWSHeartbeat();
|
||||
try {
|
||||
socket.close();
|
||||
} catch {
|
||||
// Already closing/closed - the "close" handler (if it hasn't
|
||||
// already run) will still fire and take care of reconnecting.
|
||||
}
|
||||
return;
|
||||
}
|
||||
try {
|
||||
socket.send(JSON.stringify({
|
||||
"@type": "Request",
|
||||
requestId: `ws-heartbeat-${Date.now()}`,
|
||||
using: ["urn:ietf:params:jmap:core"],
|
||||
methodCalls: [["Core/echo", {}, "0"]],
|
||||
}));
|
||||
} catch {
|
||||
// send() failing means the socket is already dead - the activity
|
||||
// timeout above will catch it on the next tick if "close" doesn't
|
||||
// fire first.
|
||||
}
|
||||
}, JMAPClient.WS_HEARTBEAT_INTERVAL);
|
||||
}
|
||||
|
||||
private stopWSHeartbeat(): void {
|
||||
if (this.wsHeartbeatTimer) {
|
||||
clearInterval(this.wsHeartbeatTimer);
|
||||
this.wsHeartbeatTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Slow poll of the session's shared/secondary accounts, run in parallel with
|
||||
* SSE (which never reports them). Skipped when there are no shared accounts,
|
||||
@@ -6210,6 +6590,23 @@ export class JMAPClient implements IJMAPClient {
|
||||
);
|
||||
}
|
||||
|
||||
// Contacts and files get no push at all today (mail-index's event-driven
|
||||
// reindex depends on this poll to notice them when SSE/WS isn't
|
||||
// available) - mirrors the Calendar branch above, same accountId caveat.
|
||||
if (this.supportsContacts()) {
|
||||
using.push('urn:ietf:params:jmap:contacts');
|
||||
methodCalls.push(
|
||||
['ContactCard/get', { accountId: this.getContactsAccountId(), ids: [], properties: ['id'] }, 'f'],
|
||||
);
|
||||
}
|
||||
|
||||
if (this.hasCapability('urn:ietf:params:jmap:filenode')) {
|
||||
using.push('urn:ietf:params:jmap:filenode');
|
||||
methodCalls.push(
|
||||
['FileNode/get', { accountId: this.getFilesAccountId(), ids: [], properties: ['id'] }, 'g'],
|
||||
);
|
||||
}
|
||||
|
||||
return { using, methodCalls };
|
||||
}
|
||||
|
||||
@@ -6310,6 +6707,27 @@ export class JMAPClient implements IJMAPClient {
|
||||
this.eventSource = null;
|
||||
}
|
||||
this.stopSSEPingMonitor();
|
||||
if (this.wsReconnectTimeout) {
|
||||
clearTimeout(this.wsReconnectTimeout);
|
||||
this.wsReconnectTimeout = null;
|
||||
}
|
||||
this.stopWSHeartbeat();
|
||||
if (this.ws) {
|
||||
// Null out this.ws BEFORE close() so the "close" event handler's
|
||||
// isCurrent() check (this.ws === socket) sees a mismatch once the
|
||||
// event fires and skips scheduling a reconnect - this is an
|
||||
// intentional teardown, not a dropped connection.
|
||||
const socket = this.ws;
|
||||
this.ws = null;
|
||||
try {
|
||||
socket.close();
|
||||
} catch {
|
||||
// Already closing/closed.
|
||||
}
|
||||
}
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.wsConsecutiveFailures = 0;
|
||||
this.wsPermanentlyDisabled = false;
|
||||
this.cleanupBrowserEventListeners();
|
||||
this.stateChangeCallback = null;
|
||||
this.pollingStates = {};
|
||||
@@ -6350,7 +6768,22 @@ export class JMAPClient implements IJMAPClient {
|
||||
|
||||
if (typeof window !== 'undefined') {
|
||||
this.onlineHandler = () => {
|
||||
// Network reconnected - reconnect SSE or force a poll
|
||||
// Network reconnected - reconnect WS/SSE or force a poll. Don't
|
||||
// make the user wait through whatever backoff delay was already in
|
||||
// flight from repeated failures while offline - the network is
|
||||
// confirmed back, so retry immediately.
|
||||
const wsUrl = this.effectiveWebSocketUrl();
|
||||
if (wsUrl) {
|
||||
if (!this.ws) {
|
||||
if (this.wsReconnectTimeout) {
|
||||
clearTimeout(this.wsReconnectTimeout);
|
||||
this.wsReconnectTimeout = null;
|
||||
}
|
||||
this.wsReconnectAttempts = 0;
|
||||
this.connectWebSocket(wsUrl);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const eventSourceUrl = this.getEventSourceUrl();
|
||||
if (eventSourceUrl && !this.sseAbortController) {
|
||||
this.connectSSE(eventSourceUrl);
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
// Renderer-side client for the encrypted local search index.
|
||||
//
|
||||
// The index is EVENT-DRIVEN: the renderer already holds the live JMAP push
|
||||
// connection (WebSocket -> SSE -> polling, `lib/jmap/client.ts`'s
|
||||
// setupPushNotifications), so the moment a StateChange announces new mail, a
|
||||
// calendar change, a contact edit or a file upload, this posts to the reindex
|
||||
// route. No polling loop, no background worker, no long-lived credential -
|
||||
// just one more authenticated fetch from the place the push already arrives.
|
||||
//
|
||||
// Every function here is best-effort and never throws: a search index failing
|
||||
// to update must never break the mail UI.
|
||||
|
||||
import { apiFetch } from '@/lib/browser-navigation';
|
||||
import { debug } from '@/lib/debug';
|
||||
import type { StateChange } from '@/lib/jmap/types';
|
||||
|
||||
export type IndexContentType = 'mail' | 'calendar' | 'contact' | 'file';
|
||||
|
||||
export interface IndexRunResult {
|
||||
ok: boolean;
|
||||
written?: Partial<Record<IndexContentType, number>>;
|
||||
skipped?: IndexContentType[];
|
||||
errors?: Array<{ contentType: IndexContentType; message: string }>;
|
||||
durationMs?: number;
|
||||
/** Set when the feature isn't available (not the desktop shell, no keyring, no binding). */
|
||||
unavailable?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps JMAP `StateChange` type keys onto our content types.
|
||||
*
|
||||
* The transport is already type-generic - the WebSocket handler
|
||||
* (`client.ts:6308-6316`) and the SSE handler (`:6505`) pass the whole
|
||||
* `changed` map through untouched, and the WS subscribes with
|
||||
* `dataTypes: null` (every type) - so anything the server pushes arrives here.
|
||||
*
|
||||
* `Mailbox` is deliberately NOT mapped: a Mailbox state change is usually just
|
||||
* an unread-count move, and it fires constantly. `Email` covers the cases that
|
||||
* change indexable content.
|
||||
*/
|
||||
const STATE_TYPE_TO_CONTENT: Record<string, IndexContentType> = {
|
||||
Email: 'mail',
|
||||
Calendar: 'calendar',
|
||||
CalendarEvent: 'calendar',
|
||||
ContactCard: 'contact',
|
||||
AddressBook: 'contact',
|
||||
FileNode: 'file',
|
||||
};
|
||||
|
||||
export function contentTypesFromStateChange(change: StateChange): IndexContentType[] {
|
||||
const out = new Set<IndexContentType>();
|
||||
for (const perAccount of Object.values(change.changed ?? {})) {
|
||||
for (const stateType of Object.keys(perAccount ?? {})) {
|
||||
const mapped = STATE_TYPE_TO_CONTENT[stateType];
|
||||
if (mapped) out.add(mapped);
|
||||
}
|
||||
}
|
||||
return [...out];
|
||||
}
|
||||
|
||||
export interface IndexRequestOptions {
|
||||
types?: readonly IndexContentType[];
|
||||
/**
|
||||
* Per-type ids to index. Supply them whenever the renderer already knows
|
||||
* which objects changed - it turns the call into a couple of `Foo/get`s
|
||||
* instead of a windowed query. Mail is the frequent case and the one where
|
||||
* this matters.
|
||||
*/
|
||||
ids?: Partial<Record<IndexContentType, string[]>>;
|
||||
/** Backfill the recent window for every supported type, and prune. */
|
||||
catchUp?: boolean;
|
||||
/** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */
|
||||
slot?: number;
|
||||
}
|
||||
|
||||
let inFlight: Promise<IndexRunResult> | null = null;
|
||||
/** Set once the server says the feature isn't there, so we stop asking. */
|
||||
let knownUnavailable = false;
|
||||
|
||||
/**
|
||||
* Posts one index request. Single-flighted: a burst of deliveries coalesces
|
||||
* into the in-flight call rather than queueing N overlapping SQLite writers.
|
||||
*/
|
||||
export async function requestIndex(options: IndexRequestOptions = {}): Promise<IndexRunResult> {
|
||||
if (knownUnavailable) return { ok: false, unavailable: true };
|
||||
if (inFlight) return inFlight;
|
||||
|
||||
const query = typeof options.slot === 'number' ? `?slot=${options.slot}` : '';
|
||||
const run = (async (): Promise<IndexRunResult> => {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/reindex${query}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
types: options.types,
|
||||
ids: options.ids,
|
||||
catchUp: options.catchUp === true,
|
||||
}),
|
||||
});
|
||||
|
||||
// 404 = not the desktop shell (or the feature is gated off). Permanent for
|
||||
// this page load; stop asking so a busy mailbox doesn't post per delivery.
|
||||
if (response.status === 404) {
|
||||
knownUnavailable = true;
|
||||
return { ok: false, unavailable: true };
|
||||
}
|
||||
if (response.status === 503) {
|
||||
// No keyring / no native binding / no key channel. Also permanent for
|
||||
// this session, and the message is worth surfacing in Settings.
|
||||
knownUnavailable = true;
|
||||
const body = await response.json().catch(() => ({}));
|
||||
return { ok: false, unavailable: true, error: body?.error };
|
||||
}
|
||||
if (!response.ok) {
|
||||
const body = await response.json().catch(() => ({}));
|
||||
return { ok: false, error: body?.error || `HTTP ${response.status}` };
|
||||
}
|
||||
const body = await response.json();
|
||||
debug.log('push', '[index] reindex done', body?.written, body?.errors);
|
||||
return {
|
||||
ok: true,
|
||||
written: body?.written,
|
||||
skipped: body?.skipped,
|
||||
errors: body?.errors,
|
||||
durationMs: body?.durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
||||
} finally {
|
||||
inFlight = null;
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
/**
|
||||
* The event-driven entry point, called from the push handler.
|
||||
*
|
||||
* `mailIds` lets the caller hand over the ids it already has (the refreshed
|
||||
* mailbox page), so the frequent mail case costs one `Email/get` rather than a
|
||||
* 30-day query. The other three types are rare events (a contact edit, a file
|
||||
* upload, a calendar change), so they fall back to their own bounded queries.
|
||||
*/
|
||||
export function indexOnStateChange(
|
||||
change: StateChange,
|
||||
opts: { mailIds?: string[]; slot?: number } = {},
|
||||
): void {
|
||||
if (knownUnavailable) return;
|
||||
const types = contentTypesFromStateChange(change);
|
||||
if (types.length === 0) return;
|
||||
|
||||
const ids: Partial<Record<IndexContentType, string[]>> = {};
|
||||
if (types.includes('mail') && opts.mailIds && opts.mailIds.length > 0) {
|
||||
ids.mail = opts.mailIds.slice(0, 100);
|
||||
}
|
||||
|
||||
// Fire-and-forget on purpose: this runs inside the push handler, and the mail
|
||||
// UI must not wait on a search index.
|
||||
void requestIndex({ types, ids: Object.keys(ids).length > 0 ? ids : undefined, slot: opts.slot });
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch-time catch-up: backfills whatever changed while the app was closed,
|
||||
* for which no push event was ever delivered. Also the recovery path for the
|
||||
* polling transport, which has no signal for contacts or files at all
|
||||
* (`client.ts`'s buildStatePollingRequest polls Mailbox/Email/Calendar/
|
||||
* CalendarEvent/SieveScript only).
|
||||
*/
|
||||
export async function catchUpIndex(slot?: number): Promise<IndexRunResult> {
|
||||
return requestIndex({ catchUp: true, slot });
|
||||
}
|
||||
|
||||
export interface IndexStats {
|
||||
contentType: string;
|
||||
count: number;
|
||||
newest: string | null;
|
||||
indexedAt: number | null;
|
||||
}
|
||||
|
||||
/** Reads per-type counts without searching. Used by the Settings panel. */
|
||||
export async function fetchIndexStats(slot?: number): Promise<IndexStats[] | null> {
|
||||
const slotQuery = typeof slot === 'number' ? `&slot=${slot}` : '';
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/search?stats=true${slotQuery}`);
|
||||
if (!response.ok) return null;
|
||||
const body = await response.json();
|
||||
return Array.isArray(body?.stats) ? (body.stats as IndexStats[]) : [];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resets the "don't ask again" latch - e.g. after the user signs in again. */
|
||||
export function resetIndexAvailability(): void {
|
||||
knownUnavailable = false;
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type {
|
||||
CalendarEvent, CalendarParticipant, ContactCard, Email, EmailBodyPart, FileNode,
|
||||
} from '@/lib/jmap/types';
|
||||
import {
|
||||
contactDisplayName, emailBodyText, extractCalendarEvent, extractContact, extractFile,
|
||||
extractMail, htmlToText, MAX_BODY_CHARS, normaliseText,
|
||||
} from '../extract';
|
||||
import { buildFilePaths } from '../jmap';
|
||||
|
||||
describe('htmlToText', () => {
|
||||
it('drops script and style CONTENT, not just the tags', () => {
|
||||
// The important case: a naive `<[^>]+>` strip leaves the script body behind
|
||||
// as searchable text, so a page full of JS would pollute the index.
|
||||
const out = htmlToText('<p>Hello</p><script>var secretToken = "abc123";</script><style>.a{color:red}</style>');
|
||||
expect(out).toContain('Hello');
|
||||
expect(out).not.toContain('secretToken');
|
||||
expect(out).not.toContain('abc123');
|
||||
expect(out).not.toContain('color:red');
|
||||
});
|
||||
|
||||
it('turns block boundaries into newlines and decodes entities', () => {
|
||||
expect(htmlToText('<p>one</p><p>two</p>')).toBe('one\ntwo');
|
||||
expect(htmlToText('a<br>b')).toBe('a\nb');
|
||||
expect(htmlToText('R&D <tag> "q" x')).toBe('R&D <tag> "q" x');
|
||||
expect(htmlToText('€10 €20')).toBe('€10 €20');
|
||||
});
|
||||
|
||||
it('ignores comments and out-of-range numeric entities without throwing', () => {
|
||||
expect(htmlToText('a<!-- hidden -->b')).toBe('a b');
|
||||
expect(() => htmlToText('� �')).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('normaliseText', () => {
|
||||
it('collapses runs of spaces, tabs and non-breaking spaces', () => {
|
||||
expect(normaliseText('a \t b')).toBe('a b');
|
||||
});
|
||||
it('caps blank-line runs and handles null/undefined', () => {
|
||||
expect(normaliseText('a\n\n\n\n\nb')).toBe('a\n\nb');
|
||||
expect(normaliseText(undefined)).toBe('');
|
||||
expect(normaliseText(null)).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
function baseEmail(overrides: Partial<Email> = {}): Email {
|
||||
return {
|
||||
id: 'M1', threadId: 'T1', mailboxIds: { mb1: true }, keywords: {},
|
||||
size: 100, receivedAt: '2026-08-01T10:00:00Z', hasAttachment: false,
|
||||
...overrides,
|
||||
} as Email;
|
||||
}
|
||||
|
||||
describe('emailBodyText', () => {
|
||||
it('prefers the text/plain part', () => {
|
||||
const email = baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
htmlBody: [{ partId: 'p2' } as EmailBodyPart],
|
||||
bodyValues: { p1: { value: 'plain wins' }, p2: { value: '<b>html loses</b>' } },
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('plain wins');
|
||||
});
|
||||
|
||||
it('falls back to flattened HTML when there is no plain alternative', () => {
|
||||
const email = baseEmail({
|
||||
htmlBody: [{ partId: 'p2' } as EmailBodyPart],
|
||||
bodyValues: { p2: { value: '<p>hello</p><p>world</p>' } },
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('hello\nworld');
|
||||
});
|
||||
|
||||
it('falls back to preview when bodyValues is missing entirely', () => {
|
||||
// This is the shape a caller gets when the Email/get omitted
|
||||
// fetchTextBodyValues - a silent empty body if we did not handle it.
|
||||
const email = baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
preview: 'server preview text',
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('server preview text');
|
||||
});
|
||||
|
||||
it('treats a whitespace-only plain part as absent', () => {
|
||||
const email = baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
htmlBody: [{ partId: 'p2' } as EmailBodyPart],
|
||||
bodyValues: { p1: { value: ' \n ' }, p2: { value: 'real content' } },
|
||||
});
|
||||
expect(emailBodyText(email)).toBe('real content');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractMail', () => {
|
||||
it('flattens addresses into `people` and keeps metadata', () => {
|
||||
const doc = extractMail('acc1', baseEmail({
|
||||
subject: 'Quarterly budget',
|
||||
from: [{ name: 'Sophie Müller', email: 'sophie@example.com' }],
|
||||
to: [{ email: 'me@example.com' }],
|
||||
cc: [{ name: 'Bob', email: 'bob@example.com' }],
|
||||
preview: 'hi',
|
||||
}));
|
||||
expect(doc.contentType).toBe('mail');
|
||||
expect(doc.title).toBe('Quarterly budget');
|
||||
expect(doc.people).toContain('Sophie Müller sophie@example.com');
|
||||
expect(doc.people).toContain('bob@example.com');
|
||||
expect(doc.occurredAt).toBe('2026-08-01T10:00:00Z');
|
||||
expect(doc.metadata.threadId).toBe('T1');
|
||||
expect(doc.metadata.mailboxIds).toEqual(['mb1']);
|
||||
});
|
||||
|
||||
it('substitutes a placeholder title rather than indexing an empty one', () => {
|
||||
expect(extractMail('acc1', baseEmail()).title).toBe('(no subject)');
|
||||
});
|
||||
|
||||
it('clamps a huge body', () => {
|
||||
const doc = extractMail('acc1', baseEmail({
|
||||
textBody: [{ partId: 'p1' } as EmailBodyPart],
|
||||
bodyValues: { p1: { value: 'x'.repeat(MAX_BODY_CHARS * 2) } },
|
||||
}));
|
||||
expect(doc.body.length).toBe(MAX_BODY_CHARS);
|
||||
});
|
||||
});
|
||||
|
||||
function baseEvent(overrides: Partial<CalendarEvent> = {}): CalendarEvent {
|
||||
return {
|
||||
id: 'E1', calendarIds: { c1: true }, isDraft: false, isOrigin: true,
|
||||
utcStart: '2026-08-10T09:00:00Z', utcEnd: '2026-08-10T10:00:00Z',
|
||||
'@type': 'Event', uid: 'u1', title: 'Standup', description: '',
|
||||
descriptionContentType: 'text/plain', created: null, updated: '2026-08-01T00:00:00Z',
|
||||
sequence: 0, start: '2026-08-10T11:00:00', duration: 'PT1H', timeZone: 'Europe/Zurich',
|
||||
showWithoutTime: false, status: 'confirmed', freeBusyStatus: 'busy', privacy: 'public',
|
||||
color: null, keywords: null, categories: null, locale: null, replyTo: null,
|
||||
organizerCalendarAddress: null, participants: null, mayInviteSelf: false,
|
||||
mayInviteOthers: false, hideAttendees: false, recurrenceId: null,
|
||||
recurrenceIdTimeZone: null, recurrenceRules: null, recurrenceOverrides: null,
|
||||
excludedRecurrenceRules: null, useDefaultAlerts: false, alerts: null,
|
||||
locations: null, virtualLocations: null, links: null, relatedTo: null,
|
||||
...overrides,
|
||||
} as CalendarEvent;
|
||||
}
|
||||
|
||||
describe('extractCalendarEvent', () => {
|
||||
it('indexes description, location, attendees and organizer', () => {
|
||||
const doc = extractCalendarEvent('acc1', baseEvent({
|
||||
title: 'Lease decision',
|
||||
description: 'Zurich office lease renewal',
|
||||
locations: { l1: { '@type': 'Location', name: 'Room 3.14', description: null, locationTypes: null, coordinates: null, timeZone: null, links: null, relativeTo: null } },
|
||||
organizerCalendarAddress: 'mailto:boss@example.com',
|
||||
// A partial participant on purpose: servers omit most JSCalendar fields,
|
||||
// and the extractor must cope with exactly this shape.
|
||||
participants: {
|
||||
p1: { name: 'Ana', email: 'ana@example.com', sendTo: { imip: 'mailto:ana@example.com' } } as unknown as CalendarParticipant,
|
||||
},
|
||||
}));
|
||||
expect(doc.title).toBe('Lease decision');
|
||||
expect(doc.body).toContain('Zurich office lease renewal');
|
||||
expect(doc.body).toContain('Room 3.14');
|
||||
// mailto: prefixes stripped so the address tokenises like every other one.
|
||||
expect(doc.people).toContain('boss@example.com');
|
||||
expect(doc.people).not.toContain('mailto:');
|
||||
expect(doc.people).toContain('ana@example.com');
|
||||
expect(doc.metadata.participantCount).toBe(1);
|
||||
});
|
||||
|
||||
it('flattens an HTML description', () => {
|
||||
const doc = extractCalendarEvent('acc1', baseEvent({
|
||||
description: '<p>agenda</p><script>bad()</script>',
|
||||
descriptionContentType: 'text/html',
|
||||
}));
|
||||
expect(doc.body).toContain('agenda');
|
||||
expect(doc.body).not.toContain('bad()');
|
||||
});
|
||||
|
||||
it('prefers utcStart over the zone-less local start for ordering', () => {
|
||||
expect(extractCalendarEvent('acc1', baseEvent()).occurredAt).toBe('2026-08-10T09:00:00Z');
|
||||
expect(extractCalendarEvent('acc1', baseEvent({ utcStart: null })).occurredAt)
|
||||
.toBe('2026-08-10T11:00:00');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractContact', () => {
|
||||
const card = (overrides: Partial<ContactCard> = {}): ContactCard =>
|
||||
({ id: 'C1', addressBookIds: { a1: true }, ...overrides }) as ContactCard;
|
||||
|
||||
it('uses name.full when present', () => {
|
||||
expect(contactDisplayName(card({ name: { full: 'Ada Lovelace' } }))).toBe('Ada Lovelace');
|
||||
});
|
||||
|
||||
it('assembles components in the right order when full is absent', () => {
|
||||
expect(contactDisplayName(card({
|
||||
name: { components: [{ kind: 'surname', value: 'Hopper' }, { kind: 'given', value: 'Grace' }] },
|
||||
}))).toBe('Grace Hopper');
|
||||
});
|
||||
|
||||
it('degrades to an email, then an org, then a placeholder', () => {
|
||||
expect(contactDisplayName(card({ emails: { e: { address: 'x@y.z' } } }))).toBe('x@y.z');
|
||||
expect(contactDisplayName(card({ organizations: { o: { name: 'ACME' } } }))).toBe('ACME');
|
||||
expect(contactDisplayName(card())).toBe('(unnamed contact)');
|
||||
});
|
||||
|
||||
it('puts emails and phones in `people` and notes/orgs in `body`', () => {
|
||||
const doc = extractContact('acc1', card({
|
||||
name: { full: 'Ada Lovelace' },
|
||||
emails: { e1: { address: 'ada@example.com' } },
|
||||
phones: { p1: { number: '+41 44 000 00 00' } },
|
||||
organizations: { o1: { name: 'Analytical Engines' } },
|
||||
notes: { n1: { note: 'met at the Zurich conference' } },
|
||||
nicknames: { k1: { name: 'The Countess' } },
|
||||
}));
|
||||
expect(doc.people).toContain('ada@example.com');
|
||||
expect(doc.people).toContain('+41 44 000 00 00');
|
||||
expect(doc.people).toContain('The Countess');
|
||||
expect(doc.body).toContain('Analytical Engines');
|
||||
expect(doc.body).toContain('met at the Zurich conference');
|
||||
// A contact has no single meaningful date; ranking is relevance-only.
|
||||
expect(doc.occurredAt).toBeNull();
|
||||
});
|
||||
|
||||
it('handles both RFC 9553 and legacy flat address shapes', () => {
|
||||
expect(extractContact('acc1', card({ addresses: { a: { full: 'Bahnhofstrasse 1, Zurich' } } })).body)
|
||||
.toContain('Bahnhofstrasse 1, Zurich');
|
||||
expect(extractContact('acc1', card({ addresses: { a: { street: 'Bahnhofstrasse 1', locality: 'Zurich' } } })).body)
|
||||
.toContain('Bahnhofstrasse 1, Zurich');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractFile', () => {
|
||||
const node = (overrides: Partial<FileNode> = {}): FileNode =>
|
||||
({
|
||||
id: 'F1', parentId: null, name: 'invoice.pdf', type: 'application/pdf',
|
||||
blobId: 'b1', size: 1234, created: '2026-07-01T00:00:00Z',
|
||||
modified: '2026-07-15T00:00:00Z', ...overrides,
|
||||
}) as FileNode;
|
||||
|
||||
it('indexes metadata only and says so', () => {
|
||||
const doc = extractFile('acc1', node(), { path: 'Finance/2026' });
|
||||
expect(doc.title).toBe('invoice.pdf');
|
||||
expect(doc.body).toContain('Finance/2026');
|
||||
expect(doc.body).toContain('pdf');
|
||||
expect(doc.metadata.contentIndexed).toBe(false);
|
||||
expect(doc.metadata.mimeType).toBe('application/pdf');
|
||||
expect(doc.metadata.size).toBe(1234);
|
||||
});
|
||||
|
||||
it('uses `modified` (FileNode has no `updated`) and falls back to `created`', () => {
|
||||
expect(extractFile('acc1', node()).occurredAt).toBe('2026-07-15T00:00:00Z');
|
||||
expect(extractFile('acc1', node({ modified: undefined as unknown as string })).occurredAt)
|
||||
.toBe('2026-07-01T00:00:00Z');
|
||||
});
|
||||
|
||||
it('marks directories', () => {
|
||||
const doc = extractFile('acc1', node({ name: 'Finance', type: 'd', blobId: null }));
|
||||
expect(doc.metadata.isDirectory).toBe(true);
|
||||
expect(doc.metadata.mimeType).toBeNull();
|
||||
expect(doc.body).toContain('folder');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildFilePaths', () => {
|
||||
it('resolves the PARENT chain, excluding the node itself', () => {
|
||||
const nodes = [
|
||||
{ id: 'root', parentId: null, name: 'Finance' },
|
||||
{ id: 'year', parentId: 'root', name: '2026' },
|
||||
{ id: 'file', parentId: 'year', name: 'invoice.pdf' },
|
||||
] as FileNode[];
|
||||
const paths = buildFilePaths(nodes);
|
||||
expect(paths.get('file')).toBe('Finance/2026');
|
||||
expect(paths.get('year')).toBe('Finance');
|
||||
expect(paths.get('root')).toBe('');
|
||||
});
|
||||
|
||||
it('truncates rather than failing when an ancestor is not in the set', () => {
|
||||
const nodes = [{ id: 'file', parentId: 'missing', name: 'x.txt' }] as FileNode[];
|
||||
expect(buildFilePaths(nodes).get('file')).toBe('');
|
||||
});
|
||||
|
||||
it('terminates on a parent cycle', () => {
|
||||
const nodes = [
|
||||
{ id: 'a', parentId: 'b', name: 'A' },
|
||||
{ id: 'b', parentId: 'a', name: 'B' },
|
||||
] as FileNode[];
|
||||
expect(() => buildFilePaths(nodes)).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,261 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { isSqlcipherAvailable } from '../binding';
|
||||
import { accountFileToken, getStoreDir, indexDbPath, STORE_DIR_ENV } from '../paths';
|
||||
import { MailIndex, toFtsMatchQuery, type IndexDoc } from '../store';
|
||||
|
||||
describe('toFtsMatchQuery', () => {
|
||||
it('quotes every token so FTS5 operators in user input cannot break the query', () => {
|
||||
// FTS5's MATCH grammar is NOT protected by SQL parameter binding: a bare
|
||||
// quote or a stray NEAR/AND/* raises `fts5: syntax error`, which would turn
|
||||
// a search box into a 500.
|
||||
expect(toFtsMatchQuery('a" OR b')).toBe('"a" AND "OR" AND "b"');
|
||||
// No trailing `*` here: the final token is one character, below the
|
||||
// prefix-match threshold (see the next test).
|
||||
expect(toFtsMatchQuery('NEAR(x y)')).toBe('"NEAR" AND "x" AND "y"');
|
||||
expect(toFtsMatchQuery('NEAR(x yes)')).toBe('"NEAR" AND "x" AND "yes"*');
|
||||
expect(toFtsMatchQuery('foo*')).toBe('"foo"*');
|
||||
expect(toFtsMatchQuery('a AND NOT b')).toContain('"NOT"');
|
||||
});
|
||||
|
||||
it('prefix-matches only the final token, and only when it is long enough', () => {
|
||||
expect(toFtsMatchQuery('zurich lea')).toBe('"zurich" AND "lea"*');
|
||||
// Two characters would match too much of a mailbox to be useful.
|
||||
expect(toFtsMatchQuery('zurich le')).toBe('"zurich" AND "le"');
|
||||
});
|
||||
|
||||
it('keeps unicode letters, emails and hyphenated words', () => {
|
||||
expect(toFtsMatchQuery('Müller')).toBe('"Müller"*');
|
||||
expect(toFtsMatchQuery('東京')).toBe('"東京"');
|
||||
expect(toFtsMatchQuery('a@b.com')).toBe('"a@b.com"*');
|
||||
expect(toFtsMatchQuery("O'Brien-Smith")).toBe('"O\'Brien-Smith"*');
|
||||
});
|
||||
|
||||
it('returns null for input with no usable tokens', () => {
|
||||
expect(toFtsMatchQuery('')).toBeNull();
|
||||
expect(toFtsMatchQuery(' ')).toBeNull();
|
||||
expect(toFtsMatchQuery('***')).toBeNull();
|
||||
expect(toFtsMatchQuery(undefined as unknown as string)).toBeNull();
|
||||
});
|
||||
|
||||
it('bounds the token count', () => {
|
||||
const many = Array.from({ length: 100 }, (_, i) => `w${i}`).join(' ');
|
||||
expect((toFtsMatchQuery(many) ?? '').split(' AND ')).toHaveLength(24);
|
||||
});
|
||||
});
|
||||
|
||||
describe('paths', () => {
|
||||
const original = process.env[STORE_DIR_ENV];
|
||||
afterEach(() => {
|
||||
if (original === undefined) delete process.env[STORE_DIR_ENV];
|
||||
else process.env[STORE_DIR_ENV] = original;
|
||||
});
|
||||
|
||||
it('is disabled unless the env var is set - the hosted-deployment gate', () => {
|
||||
delete process.env[STORE_DIR_ENV];
|
||||
expect(getStoreDir()).toBeNull();
|
||||
process.env[STORE_DIR_ENV] = '';
|
||||
expect(getStoreDir()).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects a relative path, which would resolve against the server cwd', () => {
|
||||
process.env[STORE_DIR_ENV] = 'offline';
|
||||
expect(getStoreDir()).toBeNull();
|
||||
process.env[STORE_DIR_ENV] = '/abs/offline';
|
||||
expect(getStoreDir()).toBe('/abs/offline');
|
||||
});
|
||||
|
||||
it('hashes the filename so the directory is not an account inventory', () => {
|
||||
const token = accountFileToken('linus@example.com');
|
||||
expect(token).toMatch(/^[0-9a-f]{32}$/);
|
||||
expect(token).not.toContain('linus');
|
||||
expect(indexDbPath('/s', 'linus@example.com')).toBe(`/s/index/${token}.db`);
|
||||
// Deterministic - the same account must resolve to the same file forever.
|
||||
expect(accountFileToken('linus@example.com')).toBe(token);
|
||||
});
|
||||
});
|
||||
|
||||
function doc(overrides: Partial<IndexDoc> = {}): IndexDoc {
|
||||
return {
|
||||
jmapAccountId: 'acc1',
|
||||
contentType: 'mail',
|
||||
id: 'M1',
|
||||
title: 'Quarterly budget review',
|
||||
people: 'Sophie Müller sophie@example.com',
|
||||
body: 'The Zurich office lease renewal needs a decision before September.',
|
||||
occurredAt: '2026-08-01T10:00:00Z',
|
||||
metadata: { threadId: 'T1' },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// The native binding is an OPTIONAL dependency, so these skip rather than fail
|
||||
// on a platform with no prebuild (e.g. Alpine/musl in CI containers).
|
||||
describe.skipIf(!isSqlcipherAvailable())('MailIndex (real SQLCipher)', () => {
|
||||
let storeDir: string;
|
||||
const accountId = 'linus@example.com';
|
||||
const key = randomBytes(32);
|
||||
|
||||
beforeEach(() => {
|
||||
storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mail-index-test-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(storeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const open = () => MailIndex.open({ storeDir, accountId, key });
|
||||
|
||||
it('writes an ENCRYPTED file - no plaintext recoverable from the raw bytes', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
index.close();
|
||||
|
||||
const bytes = fs.readFileSync(indexDbPath(storeDir, accountId));
|
||||
// The canary check, not just a header check: this is the assertion that
|
||||
// would have caught `PRAGMA key` being a silent no-op.
|
||||
expect(bytes.includes('Zurich office lease')).toBe(false);
|
||||
expect(bytes.includes('Quarterly budget')).toBe(false);
|
||||
expect(bytes.subarray(0, 15).toString('latin1')).not.toBe('SQLite format 3');
|
||||
});
|
||||
|
||||
it('rejects a wrong key and rebuilds instead of throwing at the caller', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
index.close();
|
||||
|
||||
// A different key cannot read the data; the store recreates the file rather
|
||||
// than surfacing an unrecoverable error, because the index is derived data
|
||||
// and the key was never a user secret.
|
||||
const other = MailIndex.open({ storeDir, accountId, key: randomBytes(32) });
|
||||
expect(other.search({ query: 'Zurich' })).toHaveLength(0);
|
||||
other.close();
|
||||
});
|
||||
|
||||
it('refuses a key of the wrong length', () => {
|
||||
expect(() => MailIndex.open({ storeDir, accountId, key: randomBytes(16) })).toThrow(/32 bytes/);
|
||||
});
|
||||
|
||||
it('finds documents by body, title and people', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
expect(index.search({ query: 'Zurich' }).map((h) => h.id)).toEqual(['M1']);
|
||||
expect(index.search({ query: 'quarterly' }).map((h) => h.id)).toEqual(['M1']);
|
||||
expect(index.search({ query: 'sophie@example.com' }).map((h) => h.id)).toEqual(['M1']);
|
||||
expect(index.search({ query: 'nonexistentword' })).toHaveLength(0);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('returns a snippet for use as LLM context', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
const [hit] = index.search({ query: 'Zurich' });
|
||||
expect(hit.snippet).toContain('[Zurich]');
|
||||
expect(hit.metadata.threadId).toBe('T1');
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('upserting the same id REPLACES the FTS row rather than duplicating it', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
index.upsert([doc({ body: 'Completely different content about Geneva.' })]);
|
||||
|
||||
// One row, and the OLD text must no longer match - the classic
|
||||
// stale-FTS-row bug when the index is maintained by hand.
|
||||
expect(index.search({ query: 'Geneva' })).toHaveLength(1);
|
||||
expect(index.search({ query: 'Zurich' })).toHaveLength(0);
|
||||
expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(1);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('scopes rows by JMAP account, so delegated accounts cannot merge', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ jmapAccountId: 'acc1', id: 'X', body: 'shared secret alpha' }),
|
||||
// Same JMAP id under a different account - legal, since JMAP ids are only
|
||||
// unique within an account (see namespaceMailboxIds in lib/jmap/client.ts).
|
||||
doc({ jmapAccountId: 'acc2', id: 'X', body: 'shared secret beta' }),
|
||||
]);
|
||||
expect(index.stats().find((s) => s.contentType === 'mail')?.count).toBe(2);
|
||||
const hits = index.search({ query: 'secret' });
|
||||
expect(hits).toHaveLength(2);
|
||||
expect(new Set(hits.map((h) => h.jmapAccountId))).toEqual(new Set(['acc1', 'acc2']));
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('filters by content type and searches across all four by default', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ contentType: 'mail', id: 'm', title: 'Zurich mail' }),
|
||||
doc({ contentType: 'calendar', id: 'c', title: 'Zurich meeting' }),
|
||||
doc({ contentType: 'contact', id: 'k', title: 'Zurich person', occurredAt: null }),
|
||||
doc({ contentType: 'file', id: 'f', title: 'Zurich file' }),
|
||||
]);
|
||||
expect(index.search({ query: 'Zurich' })).toHaveLength(4);
|
||||
expect(index.search({ query: 'Zurich', types: ['calendar'] }).map((h) => h.id)).toEqual(['c']);
|
||||
expect(new Set(index.search({ query: 'Zurich', types: ['mail', 'file'] }).map((h) => h.id)))
|
||||
.toEqual(new Set(['m', 'f']));
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('weights a title hit above a body-only hit', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ id: 'body-only', title: 'unrelated', body: 'mentions lease once' }),
|
||||
doc({ id: 'in-title', title: 'lease renewal', body: 'unrelated text' }),
|
||||
]);
|
||||
// bm25 is negative and lower is better, so the title hit must come first.
|
||||
expect(index.search({ query: 'lease' })[0].id).toBe('in-title');
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('removes documents and their FTS rows', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
expect(index.remove('acc1', 'mail', ['M1'])).toBe(1);
|
||||
expect(index.search({ query: 'Zurich' })).toHaveLength(0);
|
||||
expect(index.remove('acc1', 'mail', ['does-not-exist'])).toBe(0);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('prunes by date without touching newer rows', () => {
|
||||
const index = open();
|
||||
index.upsert([
|
||||
doc({ id: 'old', occurredAt: '2020-01-01T00:00:00Z', body: 'ancient lease' }),
|
||||
doc({ id: 'new', occurredAt: '2026-08-01T00:00:00Z', body: 'current lease' }),
|
||||
]);
|
||||
expect(index.pruneOlderThan('acc1', 'mail', '2026-01-01T00:00:00Z')).toBe(1);
|
||||
expect(index.search({ query: 'lease' }).map((h) => h.id)).toEqual(['new']);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('reports existing ids and per-type stats', () => {
|
||||
const index = open();
|
||||
index.upsert([doc({ id: 'a' }), doc({ id: 'b' }), doc({ contentType: 'file', id: 'f' })]);
|
||||
expect(index.existingIds('acc1', 'mail')).toEqual(new Set(['a', 'b']));
|
||||
const stats = index.stats();
|
||||
expect(stats.find((s) => s.contentType === 'mail')?.count).toBe(2);
|
||||
expect(stats.find((s) => s.contentType === 'file')?.count).toBe(1);
|
||||
index.close();
|
||||
});
|
||||
|
||||
it('survives reopening and keeps the data', () => {
|
||||
const first = open();
|
||||
first.upsert([doc()]);
|
||||
first.close();
|
||||
const second = open();
|
||||
expect(second.search({ query: 'Zurich' })).toHaveLength(1);
|
||||
second.close();
|
||||
});
|
||||
|
||||
it('tolerates a hostile query string end to end', () => {
|
||||
const index = open();
|
||||
index.upsert([doc()]);
|
||||
for (const q of ['"', '*', 'a" OR "b', 'NEAR(', ')', 'AND', '^', ':', '-']) {
|
||||
expect(() => index.search({ query: q })).not.toThrow();
|
||||
}
|
||||
index.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
// Guarded loader for the SQLCipher native binding.
|
||||
//
|
||||
// WHY THIS FILE EXISTS AT ALL: `@signalapp/sqlcipher` is declared in
|
||||
// package.json's `optionalDependencies`, not `dependencies`, and it MUST stay
|
||||
// there. It publishes six N-API prebuilds (darwin/linux/win32 x arm64/x64) and
|
||||
// **no build sources at all** - the published tarball has no `binding.gyp`, no
|
||||
// `src/`, no `deps/`. Its install script is `node-gyp-build`, which falls back
|
||||
// to `node-gyp rebuild` when no prebuild matches, and that fallback cannot
|
||||
// succeed without sources. So on a platform with no matching prebuild the
|
||||
// install FAILS.
|
||||
//
|
||||
// Both Dockerfiles in this repo are `FROM node:24-alpine` + `npm ci`
|
||||
// (`Dockerfile:1-4`, `integration/webmail.Dockerfile:15-19`). Alpine is musl;
|
||||
// there is no `linuxmusl-*` prebuild (and the glibc prebuild could not load
|
||||
// there anyway). As a hard `dependencies` entry this would break the
|
||||
// production image build and the integration fixture's webmail container -
|
||||
// neither of which wants this feature, they just need `npm ci` to exit 0.
|
||||
// `optionalDependencies` makes npm treat that install failure as non-fatal and
|
||||
// simply omit the package.
|
||||
//
|
||||
// The cost of that choice is exactly this module: the require must be guarded
|
||||
// at runtime, because "installed" is no longer guaranteed. Callers get
|
||||
// `null` and the feature turns itself off, which is the correct behaviour for
|
||||
// a desktop-only search index in a server that may not be a desktop.
|
||||
|
||||
/**
|
||||
* Minimal structural type for the bits of `@signalapp/sqlcipher` we use.
|
||||
*
|
||||
* Deliberately hand-written rather than `typeof import('@signalapp/sqlcipher')`:
|
||||
* the package is optional, so a type-only import would make `tsc` fail on any
|
||||
* machine where the install was skipped - which is every Alpine CI container.
|
||||
*
|
||||
* NOTE the parameter shape. `@signalapp/sqlcipher` is NOT drop-in compatible
|
||||
* with better-sqlite3 here: its `#checkParams` throws
|
||||
* `TypeError: Params must be either object or array`, so `stmt.run(a, b, c)`
|
||||
* (varargs, which better-sqlite3 accepts) is a runtime error. Always pass a
|
||||
* single array or object. Found by executing it, not by reading the types.
|
||||
*/
|
||||
export interface SqlcipherStatement {
|
||||
run(params?: readonly unknown[] | Record<string, unknown>): { changes: number; lastInsertRowid: number };
|
||||
get(params?: readonly unknown[] | Record<string, unknown>): Record<string, unknown> | undefined;
|
||||
all(params?: readonly unknown[] | Record<string, unknown>): Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface SqlcipherDatabase {
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): SqlcipherStatement;
|
||||
pragma(source: string): unknown;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface SqlcipherConstructor {
|
||||
new (path?: string): SqlcipherDatabase;
|
||||
}
|
||||
|
||||
let cached: SqlcipherConstructor | null | undefined;
|
||||
|
||||
/**
|
||||
* Returns the Database constructor, or `null` when the optional native binding
|
||||
* is not installed / cannot load on this platform. Never throws.
|
||||
*
|
||||
* Memoised on both outcomes so a missing binding costs one failed require per
|
||||
* process rather than one per request.
|
||||
*/
|
||||
export function loadSqlcipher(): SqlcipherConstructor | null {
|
||||
if (cached !== undefined) return cached;
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const mod = require('@signalapp/sqlcipher') as
|
||||
| { default?: SqlcipherConstructor }
|
||||
| SqlcipherConstructor;
|
||||
const ctor = (mod as { default?: SqlcipherConstructor }).default ?? (mod as SqlcipherConstructor);
|
||||
cached = typeof ctor === 'function' ? ctor : null;
|
||||
} catch {
|
||||
cached = null;
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
/** True when the local index can work at all in this process. */
|
||||
export function isSqlcipherAvailable(): boolean {
|
||||
return loadSqlcipher() !== null;
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
// PURE JMAP-object -> IndexDoc extractors.
|
||||
//
|
||||
// Deliberately free of database, network and store access so every shape
|
||||
// decision here is unit-testable on its own. The JMAP shapes are awkward
|
||||
// enough (JSContact keyed maps, JSCalendar participants, FileNode's `modified`
|
||||
// rather than `updated`) that this is where the bugs would otherwise hide.
|
||||
|
||||
import type { Email, CalendarEvent, ContactCard, FileNode, EmailAddress } from '@/lib/jmap/types';
|
||||
import type { IndexDoc } from './store';
|
||||
|
||||
/** Hard cap on indexed body text per document. Keeps one enormous mail from dominating the file. */
|
||||
export const MAX_BODY_CHARS = 32_000;
|
||||
|
||||
/**
|
||||
* Minimal HTML -> text, for mail that has no `text/plain` alternative.
|
||||
*
|
||||
* Not a sanitiser and not trying to be: this output is never rendered, only
|
||||
* tokenised by FTS5 and possibly handed to an LLM as context. The repo's
|
||||
* `dompurify` needs a DOM and this runs in Node, so a DOM-free reduction is the
|
||||
* right tool. Order matters - script/style content must go before tags are
|
||||
* stripped, or their contents would leak into the index as searchable text.
|
||||
*/
|
||||
export function htmlToText(html: string): string {
|
||||
return html
|
||||
.replace(/<!--[\s\S]*?-->/g, ' ')
|
||||
.replace(/<(script|style|head)\b[\s\S]*?<\/\1>/gi, ' ')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/(p|div|tr|li|h[1-6]|blockquote)>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/&/gi, '&')
|
||||
.replace(/</gi, '<')
|
||||
.replace(/>/gi, '>')
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/&#(\d+);/g, (_m, d: string) => {
|
||||
const code = Number(d);
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
|
||||
})
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_m, h: string) => {
|
||||
const code = parseInt(h, 16);
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
|
||||
})
|
||||
.replace(/[ \t\u00a0]+/g, ' ')
|
||||
.replace(/\s*\n\s*/g, '\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function normaliseText(s: string | null | undefined): string {
|
||||
if (!s) return '';
|
||||
return s.replace(/\r\n?/g, '\n').replace(/[ \t\u00a0]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
function clamp(s: string, max = MAX_BODY_CHARS): string {
|
||||
return s.length <= max ? s : s.slice(0, max);
|
||||
}
|
||||
|
||||
function formatAddresses(list: readonly EmailAddress[] | undefined): string {
|
||||
if (!list || list.length === 0) return '';
|
||||
return list
|
||||
.map((a) => [a.name, a.email].filter((p) => typeof p === 'string' && p.length > 0).join(' '))
|
||||
.filter((s) => s.length > 0)
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
/** Values of a JSContact/JSCalendar keyed map, in a stable order. */
|
||||
function mapValues<T>(m: Record<string, T> | null | undefined): T[] {
|
||||
if (!m || typeof m !== 'object') return [];
|
||||
return Object.keys(m).sort().map((k) => m[k]);
|
||||
}
|
||||
|
||||
function joinUnique(parts: Array<string | undefined | null>): string {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const p of parts) {
|
||||
const v = typeof p === 'string' ? p.trim() : '';
|
||||
if (!v || seen.has(v)) continue;
|
||||
seen.add(v);
|
||||
out.push(v);
|
||||
}
|
||||
return out.join(', ');
|
||||
}
|
||||
|
||||
// ── mail ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Resolves an Email's plain-text body from `bodyValues`, preferring the
|
||||
* `text/plain` alternative and falling back to flattening the HTML one.
|
||||
*
|
||||
* `textBody`/`htmlBody` reference parts by `partId`; the text itself only
|
||||
* arrives in `bodyValues` when the `Email/get` asked for it
|
||||
* (`fetchTextBodyValues` / `fetchHTMLBodyValues`). A caller that forgets that
|
||||
* gets an empty body rather than an error, which is exactly the kind of silent
|
||||
* hole worth naming here.
|
||||
*/
|
||||
export function emailBodyText(email: Email): string {
|
||||
const values = email.bodyValues ?? {};
|
||||
const fromParts = (parts: typeof email.textBody): string =>
|
||||
(parts ?? [])
|
||||
.map((p) => values[p.partId]?.value ?? '')
|
||||
.filter((v) => v.length > 0)
|
||||
.join('\n\n');
|
||||
|
||||
const plain = fromParts(email.textBody);
|
||||
if (plain.trim().length > 0) return normaliseText(plain);
|
||||
|
||||
const html = fromParts(email.htmlBody);
|
||||
if (html.trim().length > 0) return normaliseText(htmlToText(html));
|
||||
|
||||
// Last resort: the server-computed preview. Better than nothing for a search
|
||||
// index, and it costs no extra round trip.
|
||||
return normaliseText(email.preview);
|
||||
}
|
||||
|
||||
export function extractMail(jmapAccountId: string, email: Email): IndexDoc {
|
||||
const body = clamp(emailBodyText(email));
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'mail',
|
||||
id: email.id,
|
||||
title: normaliseText(email.subject) || '(no subject)',
|
||||
people: joinUnique([
|
||||
formatAddresses(email.from),
|
||||
formatAddresses(email.to),
|
||||
formatAddresses(email.cc),
|
||||
]),
|
||||
body,
|
||||
occurredAt: email.receivedAt ?? null,
|
||||
metadata: {
|
||||
threadId: email.threadId,
|
||||
from: email.from?.[0]?.email ?? null,
|
||||
fromName: email.from?.[0]?.name ?? null,
|
||||
hasAttachment: !!email.hasAttachment,
|
||||
size: email.size ?? null,
|
||||
mailboxIds: Object.keys(email.mailboxIds ?? {}),
|
||||
preview: normaliseText(email.preview).slice(0, 300),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── calendar ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function extractCalendarEvent(jmapAccountId: string, event: CalendarEvent): IndexDoc {
|
||||
const participants = mapValues(event.participants);
|
||||
const participantText = joinUnique(
|
||||
participants.flatMap((p) => [
|
||||
p?.name,
|
||||
p?.email,
|
||||
p?.calendarAddress?.replace(/^mailto:/i, ''),
|
||||
...Object.values(p?.sendTo ?? {}).map((v) =>
|
||||
typeof v === 'string' ? v.replace(/^mailto:/i, '') : '',
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
const locations = mapValues(event.locations)
|
||||
.map((l) => normaliseText(l?.name))
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
// `descriptionContentType` can legitimately be text/html.
|
||||
const rawDescription = normaliseText(event.description);
|
||||
const description = /html/i.test(event.descriptionContentType ?? '')
|
||||
? normaliseText(htmlToText(rawDescription))
|
||||
: rawDescription;
|
||||
|
||||
const keywords = Object.keys(event.keywords ?? {});
|
||||
const categories = Object.keys(event.categories ?? {});
|
||||
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'calendar',
|
||||
id: event.id,
|
||||
title: normaliseText(event.title) || '(untitled event)',
|
||||
people: joinUnique([event.organizerCalendarAddress?.replace(/^mailto:/i, ''), participantText]),
|
||||
body: clamp(
|
||||
[description, locations.join(', '), keywords.join(' '), categories.join(' ')]
|
||||
.filter((s) => s.length > 0)
|
||||
.join('\n\n'),
|
||||
),
|
||||
// `utcStart` is the resolved instant the app computes; `start` is local
|
||||
// wall-clock without a zone, so prefer utcStart for ordering.
|
||||
occurredAt: event.utcStart ?? event.start ?? null,
|
||||
metadata: {
|
||||
start: event.start ?? null,
|
||||
utcStart: event.utcStart ?? null,
|
||||
utcEnd: event.utcEnd ?? null,
|
||||
timeZone: event.timeZone ?? null,
|
||||
showWithoutTime: !!event.showWithoutTime,
|
||||
status: event.status ?? null,
|
||||
locations,
|
||||
calendarIds: Object.keys(event.calendarIds ?? {}),
|
||||
participantCount: participants.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── contacts ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function contactDisplayName(card: ContactCard): string {
|
||||
const full = normaliseText(card.name?.full);
|
||||
if (full) return full;
|
||||
const components = card.name?.components ?? [];
|
||||
const ordered = ['prefix', 'given', 'given2', 'additional', 'middle', 'surname', 'surname2', 'suffix'];
|
||||
const byKind = components
|
||||
.slice()
|
||||
.sort((a, b) => ordered.indexOf(a.kind) - ordered.indexOf(b.kind))
|
||||
.map((c) => c.value)
|
||||
.filter((v) => typeof v === 'string' && v.trim().length > 0)
|
||||
.join(' ');
|
||||
if (byKind.trim()) return normaliseText(byKind);
|
||||
const firstEmail = mapValues(card.emails)[0]?.address;
|
||||
if (firstEmail) return firstEmail;
|
||||
const org = mapValues(card.organizations)[0]?.name;
|
||||
return normaliseText(org) || '(unnamed contact)';
|
||||
}
|
||||
|
||||
export function extractContact(jmapAccountId: string, card: ContactCard): IndexDoc {
|
||||
const emails = mapValues(card.emails).map((e) => e.address).filter(Boolean);
|
||||
const phones = mapValues(card.phones).map((p) => p.number).filter(Boolean);
|
||||
const nicknames = mapValues(card.nicknames)
|
||||
.map((n) => n?.name)
|
||||
.filter((v): v is string => typeof v === 'string' && v.length > 0);
|
||||
const orgs = mapValues(card.organizations).map((o) => o.name).filter((v): v is string => !!v);
|
||||
const titles = mapValues(card.titles).map((t) => t.name).filter(Boolean);
|
||||
const notes = mapValues(card.notes).map((n) => n.note).filter(Boolean);
|
||||
// `full` (RFC 9553) when present, else the legacy flat fields vCard import
|
||||
// produces, else the ordered components. All three shapes occur in this type.
|
||||
const addresses = mapValues(card.addresses)
|
||||
.map((a) =>
|
||||
normaliseText(
|
||||
a?.full ||
|
||||
[a?.street, a?.locality, a?.region, a?.postcode, a?.country]
|
||||
.filter((p): p is string => typeof p === 'string' && p.length > 0)
|
||||
.join(', ') ||
|
||||
(a?.components ?? []).map((c) => c.value).join(' '),
|
||||
),
|
||||
)
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'contact',
|
||||
id: card.id,
|
||||
title: contactDisplayName(card),
|
||||
// Emails/phones go in `people` (weighted above body) because "who is
|
||||
// this / what's their number" is the dominant contact lookup.
|
||||
people: joinUnique([...emails, ...phones, ...nicknames]),
|
||||
body: clamp([...orgs, ...titles, ...addresses, ...notes].filter(Boolean).join('\n')),
|
||||
// A contact has no meaningful single date; JSContact `updated` is optional
|
||||
// and not on this repo's type, so leave it null and rank by relevance only.
|
||||
occurredAt: null,
|
||||
metadata: {
|
||||
kind: card.kind ?? null,
|
||||
emails,
|
||||
phones,
|
||||
organizations: orgs,
|
||||
addressBookIds: Object.keys(card.addressBookIds ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── files ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* METADATA ONLY - filename, path, dates, size, owner. Deliberately NOT file
|
||||
* content: extracting searchable text from arbitrary PDFs / office documents /
|
||||
* images is a materially bigger problem (per-format parsers, OCR, size limits,
|
||||
* untrusted-input parsing in a process holding the user's mail) and is a
|
||||
* separate piece of work. `path` is passed in by the caller because a FileNode
|
||||
* only knows its `parentId`; resolving the chain is the caller's job.
|
||||
*/
|
||||
export function extractFile(
|
||||
jmapAccountId: string,
|
||||
node: FileNode,
|
||||
opts: { path?: string; ownerName?: string } = {},
|
||||
): IndexDoc {
|
||||
const dirPath = normaliseText(opts.path);
|
||||
const isDirectory = node.type === 'd';
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'file',
|
||||
id: node.id,
|
||||
title: normaliseText(node.name) || '(unnamed file)',
|
||||
people: joinUnique([opts.ownerName, node.accountName]),
|
||||
// The path is genuinely searchable text ("that thing in Invoices/2026"),
|
||||
// and the extension is worth tokenising on its own.
|
||||
body: clamp(
|
||||
[dirPath, isDirectory ? 'folder' : node.type, fileExtension(node.name)]
|
||||
.filter((s) => s && s.length > 0)
|
||||
.join('\n'),
|
||||
),
|
||||
// FileNode has `modified`, NOT `updated` - asking for the wrong name
|
||||
// silently yields undefined (this repo hit that as #700).
|
||||
occurredAt: node.modified ?? node.created ?? null,
|
||||
metadata: {
|
||||
path: dirPath || null,
|
||||
mimeType: isDirectory ? null : node.type,
|
||||
isDirectory,
|
||||
size: typeof node.size === 'number' ? node.size : null,
|
||||
created: node.created ?? null,
|
||||
modified: node.modified ?? null,
|
||||
parentId: node.parentId ?? null,
|
||||
contentIndexed: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fileExtension(name: string | undefined): string {
|
||||
if (!name) return '';
|
||||
const i = name.lastIndexOf('.');
|
||||
return i > 0 && i < name.length - 1 ? name.slice(i + 1).toLowerCase() : '';
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
// A deliberately tiny server-side JMAP client, used only by the indexer.
|
||||
//
|
||||
// WHY NOT REUSE lib/jmap/client.ts: that class is a 7400-line renderer object.
|
||||
// It holds credentials in instance fields, uses `btoa`, opens EventSource /
|
||||
// WebSocket push connections, and wires itself into Zustand stores and toast
|
||||
// notifications. Importing it into an API route would drag all of that into the
|
||||
// server bundle for the sake of four method calls. The existing server-side
|
||||
// JMAP code in this repo (lib/auth/verify-jmap-auth.ts) already sets the
|
||||
// precedent: plain fetch + an Authorization header.
|
||||
//
|
||||
// Everything here is stateless - the caller supplies the auth header per call,
|
||||
// so there is no resident credential and nothing to invalidate.
|
||||
|
||||
import type { CalendarEvent, ContactCard, Email, FileNode } from '@/lib/jmap/types';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 30_000;
|
||||
|
||||
export const CAP_CORE = 'urn:ietf:params:jmap:core';
|
||||
export const CAP_MAIL = 'urn:ietf:params:jmap:mail';
|
||||
export const CAP_CALENDARS = 'urn:ietf:params:jmap:calendars';
|
||||
export const CAP_CONTACTS = 'urn:ietf:params:jmap:contacts';
|
||||
export const CAP_FILENODE = 'urn:ietf:params:jmap:filenode';
|
||||
|
||||
export class JmapIndexError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.name = 'JmapIndexError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export interface JmapSessionInfo {
|
||||
apiUrl: string;
|
||||
/** Server-confirmed authenticated login (JMAP Session.username). */
|
||||
username?: string;
|
||||
primaryAccounts: Record<string, string>;
|
||||
accounts: Record<string, { name?: string; isPersonal?: boolean; accountCapabilities?: Record<string, unknown> }>;
|
||||
capabilities: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pins a URL advertised by the session to the origin we authenticated against.
|
||||
*
|
||||
* `lib/jmap/client.ts` does the same thing in its rewriteSessionUrls() for the
|
||||
* renderer's benefit. Server-side it is a security control, not a convenience:
|
||||
* we attach the user's credentials to this URL, so a session document that
|
||||
* advertised an `apiUrl` on someone else's host would turn this into a
|
||||
* credential-leaking SSRF. Keep the path and query, take the origin from the
|
||||
* server URL we were configured with.
|
||||
*/
|
||||
function pinToServerOrigin(advertised: string, serverUrl: string): string {
|
||||
const base = new URL(serverUrl);
|
||||
let target: URL;
|
||||
try {
|
||||
target = new URL(advertised, base);
|
||||
} catch {
|
||||
throw new JmapIndexError('JMAP session advertised an unusable apiUrl');
|
||||
}
|
||||
return `${base.origin}${target.pathname}${target.search}`;
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
return await fetch(url, { ...init, signal: controller.signal, redirect: 'manual' });
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
throw new JmapIndexError('JMAP request timed out', 504);
|
||||
}
|
||||
throw new JmapIndexError(`JMAP request failed: ${String(error)}`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** Stalwart 307-redirects /.well-known/jmap to /jmap/session. */
|
||||
const MAX_REDIRECTS = 3;
|
||||
|
||||
export async function fetchJmapSession(serverUrl: string, authHeader: string): Promise<JmapSessionInfo> {
|
||||
const base = serverUrl.replace(/\/+$/, '');
|
||||
const origin = new URL(base).origin;
|
||||
let currentUrl = `${base}/.well-known/jmap`;
|
||||
let response: Response | undefined;
|
||||
|
||||
// Redirects must be followed EXPLICITLY, not with `redirect: 'follow'`: we
|
||||
// attach the user's credentials to every hop, so each one has to be checked to
|
||||
// still be on the origin we authenticated against. A blind follow would hand
|
||||
// the Authorization header to whatever host a misconfigured or hostile session
|
||||
// pointed at. Same reasoning (and same bound) as lib/auth/verify-jmap-auth.ts.
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
||||
response = await fetchWithTimeout(currentUrl, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: authHeader },
|
||||
});
|
||||
if (response.status < 300 || response.status >= 400) break;
|
||||
|
||||
const location = response.headers.get('location');
|
||||
if (!location) throw new JmapIndexError('JMAP session redirect had no Location header');
|
||||
const next = new URL(location, currentUrl);
|
||||
if (next.origin !== origin) {
|
||||
throw new JmapIndexError(
|
||||
`JMAP session redirected off-origin (${next.origin}); refusing to send credentials there`,
|
||||
);
|
||||
}
|
||||
currentUrl = next.toString();
|
||||
}
|
||||
|
||||
if (!response) throw new JmapIndexError('JMAP session fetch produced no response');
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new JmapIndexError('JMAP authentication failed', 401);
|
||||
}
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
throw new JmapIndexError('Too many redirects fetching the JMAP session');
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new JmapIndexError(`JMAP session fetch failed (${response.status})`);
|
||||
}
|
||||
const raw = (await response.json().catch(() => null)) as Record<string, unknown> | null;
|
||||
if (!raw || typeof raw.apiUrl !== 'string') {
|
||||
throw new JmapIndexError('Invalid JMAP session response');
|
||||
}
|
||||
return {
|
||||
apiUrl: pinToServerOrigin(raw.apiUrl, serverUrl),
|
||||
username: typeof raw.username === 'string' ? raw.username : undefined,
|
||||
primaryAccounts: (raw.primaryAccounts as Record<string, string>) ?? {},
|
||||
accounts: (raw.accounts as JmapSessionInfo['accounts']) ?? {},
|
||||
capabilities: (raw.capabilities as Record<string, unknown>) ?? {},
|
||||
};
|
||||
}
|
||||
|
||||
type MethodCall = [string, Record<string, unknown>, string];
|
||||
|
||||
/** Raw method-response tuple, `[name, args, callId]`. `name` is 'error' on failure. */
|
||||
type MethodResponse = [string, Record<string, unknown>, string];
|
||||
|
||||
export async function jmapRequest(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
using: readonly string[],
|
||||
methodCalls: readonly MethodCall[],
|
||||
): Promise<MethodResponse[]> {
|
||||
const response = await fetchWithTimeout(session.apiUrl, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: authHeader, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ using, methodCalls }),
|
||||
});
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new JmapIndexError('JMAP authentication failed', 401);
|
||||
}
|
||||
if (response.status === 429) {
|
||||
throw new JmapIndexError('JMAP server is rate limiting', 429);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new JmapIndexError(`JMAP request failed (${response.status})`);
|
||||
}
|
||||
const data = (await response.json().catch(() => null)) as { methodResponses?: MethodResponse[] } | null;
|
||||
if (!data || !Array.isArray(data.methodResponses)) {
|
||||
throw new JmapIndexError('Invalid JMAP response envelope');
|
||||
}
|
||||
return data.methodResponses;
|
||||
}
|
||||
|
||||
function firstResult(responses: MethodResponse[], expected: string): Record<string, unknown> | null {
|
||||
for (const [name, args] of responses) {
|
||||
if (name === expected) return args;
|
||||
// A method-level error is not fatal for an INDEX: a server that doesn't
|
||||
// support one data type should not fail the whole reindex. The caller
|
||||
// treats null as "nothing to index for this type".
|
||||
if (name === 'error') return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function idsOf(args: Record<string, unknown> | null): string[] {
|
||||
const ids = args?.ids;
|
||||
return Array.isArray(ids) ? ids.filter((v): v is string => typeof v === 'string') : [];
|
||||
}
|
||||
|
||||
function listOf<T>(args: Record<string, unknown> | null): T[] {
|
||||
const list = args?.list;
|
||||
return Array.isArray(list) ? (list as T[]) : [];
|
||||
}
|
||||
|
||||
export function accountIdFor(session: JmapSessionInfo, capability: string): string | null {
|
||||
const id = session.primaryAccounts[capability];
|
||||
return typeof id === 'string' && id.length > 0 ? id : null;
|
||||
}
|
||||
|
||||
export function hasCapability(session: JmapSessionInfo, capability: string): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(session.capabilities, capability);
|
||||
}
|
||||
|
||||
/** Per-ACCOUNT capability, mirroring client.ts's supportsFiles() (#563: a server can advertise it while an account has it revoked). */
|
||||
export function accountHasCapability(
|
||||
session: JmapSessionInfo,
|
||||
accountId: string,
|
||||
capability: string,
|
||||
): boolean {
|
||||
const account = session.accounts[accountId];
|
||||
if (!account) return false;
|
||||
if (account.accountCapabilities && Object.prototype.hasOwnProperty.call(account.accountCapabilities, capability)) {
|
||||
return true;
|
||||
}
|
||||
return account.isPersonal === false;
|
||||
}
|
||||
|
||||
// ── mail ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Properties needed to build a mail IndexDoc. Bodies come via bodyValues. */
|
||||
const EMAIL_INDEX_PROPERTIES = [
|
||||
'id', 'threadId', 'mailboxIds', 'keywords', 'size', 'receivedAt',
|
||||
'from', 'to', 'cc', 'subject', 'preview', 'hasAttachment',
|
||||
'textBody', 'htmlBody', 'bodyValues',
|
||||
] as const;
|
||||
|
||||
export async function getEmailsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
maxBodyBytes: number,
|
||||
): Promise<Email[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||
['Email/get', {
|
||||
accountId,
|
||||
ids: [...ids],
|
||||
properties: [...EMAIL_INDEX_PROPERTIES],
|
||||
// Without these two the bodyValues map comes back EMPTY and every
|
||||
// indexed body would silently fall back to `preview`.
|
||||
fetchTextBodyValues: true,
|
||||
fetchHTMLBodyValues: true,
|
||||
maxBodyValueBytes: maxBodyBytes,
|
||||
}, 'g'],
|
||||
]);
|
||||
return listOf<Email>(firstResult(responses, 'Email/get'));
|
||||
}
|
||||
|
||||
export async function queryRecentEmailIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
afterIso: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||
['Email/query', {
|
||||
accountId,
|
||||
filter: { after: afterIso },
|
||||
sort: [{ property: 'receivedAt', isAscending: false }],
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
}, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'Email/query'));
|
||||
}
|
||||
|
||||
// ── calendar ────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function getCalendarEventsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<CalendarEvent[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
||||
['CalendarEvent/get', { accountId, ids: [...ids] }, 'g'],
|
||||
]);
|
||||
return listOf<CalendarEvent>(firstResult(responses, 'CalendarEvent/get'));
|
||||
}
|
||||
|
||||
export async function queryCalendarEventIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
afterIso: string,
|
||||
beforeIso: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
||||
['CalendarEvent/query', {
|
||||
accountId,
|
||||
// LocalDateTime, per the note in lib/jmap/client.ts:307-312 - Stalwart
|
||||
// parses these without a zone suffix and ignores unparseable values.
|
||||
filter: { after: toLocalDateTime(afterIso), before: toLocalDateTime(beforeIso) },
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
}, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'CalendarEvent/query'));
|
||||
}
|
||||
|
||||
/** JSCalendar LocalDateTime: `YYYY-MM-DDTHH:MM:SS`, no zone designator. */
|
||||
function toLocalDateTime(iso: string): string {
|
||||
return iso.replace(/\.\d+/, '').replace(/Z$/, '').slice(0, 19);
|
||||
}
|
||||
|
||||
// ── contacts ────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function getContactsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<ContactCard[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
||||
['ContactCard/get', { accountId, ids: [...ids] }, 'g'],
|
||||
]);
|
||||
return listOf<ContactCard>(firstResult(responses, 'ContactCard/get'));
|
||||
}
|
||||
|
||||
export async function queryContactIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
||||
['ContactCard/query', { accountId, limit, calculateTotal: false }, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'ContactCard/query'));
|
||||
}
|
||||
|
||||
// ── files ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const FILENODE_INDEX_PROPERTIES = [
|
||||
'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified',
|
||||
] as const;
|
||||
|
||||
export async function getFilesForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<FileNode[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
||||
['FileNode/get', { accountId, ids: [...ids], properties: [...FILENODE_INDEX_PROPERTIES] }, 'g'],
|
||||
]);
|
||||
return listOf<FileNode>(firstResult(responses, 'FileNode/get'));
|
||||
}
|
||||
|
||||
export async function queryFileIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
||||
['FileNode/query', { accountId, filter: {}, limit, calculateTotal: false }, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'FileNode/query'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds `id -> "Parent/Child"` paths for the given nodes, walking `parentId`
|
||||
* upward. FileNode only knows its parent, so the caller has to assemble this;
|
||||
* unresolvable ancestors just truncate the path rather than failing.
|
||||
*/
|
||||
export function buildFilePaths(nodes: readonly FileNode[]): Map<string, string> {
|
||||
const byId = new Map(nodes.map((n) => [n.id, n]));
|
||||
const cache = new Map<string, string>();
|
||||
|
||||
const resolve = (id: string, depth: number): string => {
|
||||
if (depth > 32) return '';
|
||||
const cached = cache.get(id);
|
||||
if (cached !== undefined) return cached;
|
||||
const node = byId.get(id);
|
||||
if (!node) return '';
|
||||
const parent = node.parentId ? resolve(node.parentId, depth + 1) : '';
|
||||
const full = parent ? `${parent}/${node.name}` : node.name;
|
||||
cache.set(id, full);
|
||||
return full;
|
||||
};
|
||||
|
||||
const out = new Map<string, string>();
|
||||
for (const n of nodes) {
|
||||
// The document's own `path` metadata is its PARENT directory chain, so a
|
||||
// search for "Invoices" matches files inside it without the filename
|
||||
// being duplicated into the body.
|
||||
out.set(n.id, n.parentId ? resolve(n.parentId, 0) : '');
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Server-side client for the main process's key service (electron/key-service.ts).
|
||||
//
|
||||
// Asks for an account's index key over the inherited fd only when a job needs
|
||||
// it, and drops it as soon as the job finishes. There is deliberately no cache:
|
||||
// a resident plaintext key in a long-lived process is exactly the thing the OS
|
||||
// keychain exists to avoid, and a keychain round trip costs microseconds
|
||||
// against a job that makes network calls.
|
||||
|
||||
import net from 'node:net';
|
||||
|
||||
/** Set by electron/main.ts alongside VNCMAIL_DESKTOP_STORE_DIR. */
|
||||
export const KEY_FD_ENV = 'VNCMAIL_DESKTOP_KEY_FD';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
export type KeyErrorCode =
|
||||
| 'no-channel'
|
||||
| 'no-secure-storage'
|
||||
| 'key-io-failed'
|
||||
| 'key-unreadable'
|
||||
| 'bad-request'
|
||||
| 'timeout';
|
||||
|
||||
export class IndexKeyError extends Error {
|
||||
code: KeyErrorCode;
|
||||
constructor(code: KeyErrorCode, message: string) {
|
||||
super(message);
|
||||
this.name = 'IndexKeyError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
interface Pending {
|
||||
resolve: (value: { key?: string }) => void;
|
||||
reject: (error: Error) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Channel state lives on `globalThis`, NOT in module scope.
|
||||
*
|
||||
* A file descriptor can be adopted as a socket exactly ONCE per process: a
|
||||
* second `new net.Socket({ fd })` for an fd this process already owns throws
|
||||
* `EEXIST` from libuv's uv_pipe_open. Module scope is not once-per-process -
|
||||
* Next re-evaluates route modules (dev HMR, and separate module instances
|
||||
* across route bundles), so a module-scoped `let socket` produced exactly that
|
||||
* crash: `Could not open fd 3: Error: open EEXIST`, found by the integration
|
||||
* test rather than by reading the code.
|
||||
*
|
||||
* A Symbol key on globalThis is the one place in a Node process that survives
|
||||
* module re-evaluation, so adoption genuinely happens once.
|
||||
*/
|
||||
interface ChannelState {
|
||||
socket: net.Socket | null;
|
||||
nextId: number;
|
||||
pending: Map<number, Pending>;
|
||||
readBuffer: string;
|
||||
}
|
||||
|
||||
const STATE_KEY = Symbol.for('vncmail.mailIndex.keyChannel');
|
||||
|
||||
function state(): ChannelState {
|
||||
const holder = globalThis as unknown as Record<symbol, ChannelState | undefined>;
|
||||
const existing = holder[STATE_KEY];
|
||||
if (existing) return existing;
|
||||
const created: ChannelState = { socket: null, nextId: 1, pending: new Map(), readBuffer: '' };
|
||||
holder[STATE_KEY] = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
function failAll(s: ChannelState, error: Error): void {
|
||||
for (const [, p] of s.pending) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(error);
|
||||
}
|
||||
s.pending.clear();
|
||||
}
|
||||
|
||||
function getSocket(): net.Socket {
|
||||
const s = state();
|
||||
if (s.socket && !s.socket.destroyed) return s.socket;
|
||||
|
||||
const raw = process.env[KEY_FD_ENV]?.trim();
|
||||
const fd = raw ? Number(raw) : NaN;
|
||||
if (!Number.isInteger(fd) || fd < 3) {
|
||||
throw new IndexKeyError(
|
||||
'no-channel',
|
||||
`${KEY_FD_ENV} is not a usable file descriptor (got ${JSON.stringify(raw)}). ` +
|
||||
`The local index only works inside the Electron desktop shell.`,
|
||||
);
|
||||
}
|
||||
|
||||
let created: net.Socket;
|
||||
try {
|
||||
created = new net.Socket({ fd, readable: true, writable: true });
|
||||
} catch (error) {
|
||||
throw new IndexKeyError('no-channel', `Could not open fd ${fd}: ${String(error)}`);
|
||||
}
|
||||
// The channel outlives every individual request; don't let it hold the event
|
||||
// loop open on its own.
|
||||
created.unref();
|
||||
|
||||
created.on('data', (chunk: Buffer) => {
|
||||
s.readBuffer += chunk.toString('utf8');
|
||||
if (s.readBuffer.length > 64 * 1024) s.readBuffer = '';
|
||||
let newline: number;
|
||||
while ((newline = s.readBuffer.indexOf('\n')) >= 0) {
|
||||
const line = s.readBuffer.slice(0, newline);
|
||||
s.readBuffer = s.readBuffer.slice(newline + 1);
|
||||
if (!line.trim()) continue;
|
||||
let msg: { id?: unknown; ok?: unknown; key?: unknown; code?: unknown; error?: unknown };
|
||||
try {
|
||||
msg = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const id = typeof msg.id === 'number' ? msg.id : null;
|
||||
if (id === null) continue;
|
||||
const p = s.pending.get(id);
|
||||
if (!p) continue;
|
||||
s.pending.delete(id);
|
||||
clearTimeout(p.timer);
|
||||
if (msg.ok === true) {
|
||||
p.resolve({ key: typeof msg.key === 'string' ? msg.key : undefined });
|
||||
} else {
|
||||
const code = typeof msg.code === 'string' ? (msg.code as KeyErrorCode) : 'key-io-failed';
|
||||
p.reject(new IndexKeyError(code, typeof msg.error === 'string' ? msg.error : 'Key request failed'));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const onGone = (error?: Error) => {
|
||||
s.socket = null;
|
||||
s.readBuffer = '';
|
||||
failAll(s, error ?? new IndexKeyError('no-channel', 'Key service channel closed'));
|
||||
};
|
||||
created.on('close', () => onGone());
|
||||
created.on('error', (error) => onGone(new IndexKeyError('no-channel', String(error))));
|
||||
|
||||
s.socket = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
function request(op: 'getIndexKey' | 'deleteIndexKey', accountId: string): Promise<{ key?: string }> {
|
||||
const sock = getSocket();
|
||||
const s = state();
|
||||
const id = s.nextId++;
|
||||
return new Promise<{ key?: string }>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
s.pending.delete(id);
|
||||
reject(new IndexKeyError('timeout', `Key service did not answer within ${REQUEST_TIMEOUT_MS}ms`));
|
||||
}, REQUEST_TIMEOUT_MS);
|
||||
// Don't let a pending key request keep the process alive either.
|
||||
timer.unref?.();
|
||||
s.pending.set(id, { resolve, reject, timer });
|
||||
try {
|
||||
sock.write(`${JSON.stringify({ id, op, accountId })}\n`);
|
||||
} catch (error) {
|
||||
s.pending.delete(id);
|
||||
clearTimeout(timer);
|
||||
reject(new IndexKeyError('no-channel', `Could not write to the key service: ${String(error)}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `fn` with the account's raw index key, then zeroes the buffer.
|
||||
*
|
||||
* Zeroing a Buffer is genuine (unlike a JS string, which cannot be scrubbed) -
|
||||
* which is why the key crosses the boundary as hex and is converted to a Buffer
|
||||
* exactly once, here. `store.ts` puts the hex into a `PRAGMA` string, so a copy
|
||||
* does briefly exist in the JS heap; the buffer wipe bounds how long the
|
||||
* long-lived copy lives, it does not pretend to eliminate every trace.
|
||||
*/
|
||||
export async function withIndexKey<T>(
|
||||
accountId: string,
|
||||
fn: (key: Buffer) => Promise<T> | T,
|
||||
): Promise<T> {
|
||||
const { key: hex } = await request('getIndexKey', accountId);
|
||||
if (!hex) throw new IndexKeyError('key-io-failed', 'Key service returned no key');
|
||||
const key = Buffer.from(hex, 'hex');
|
||||
if (key.length !== 32) {
|
||||
key.fill(0);
|
||||
throw new IndexKeyError('key-io-failed', `Key service returned ${key.length} bytes, expected 32`);
|
||||
}
|
||||
try {
|
||||
return await fn(key);
|
||||
} finally {
|
||||
key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
/** Used when purging an account: the key goes FIRST, so an interrupted purge leaves unreadable data. */
|
||||
export async function deleteIndexKey(accountId: string): Promise<void> {
|
||||
await request('deleteIndexKey', accountId);
|
||||
}
|
||||
|
||||
/** True when this process has a key channel at all (i.e. is the desktop shell's server). */
|
||||
export function hasKeyChannel(): boolean {
|
||||
const raw = process.env[KEY_FD_ENV]?.trim();
|
||||
const fd = raw ? Number(raw) : NaN;
|
||||
return Number.isInteger(fd) && fd >= 3;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// The hosted-deployment gate, and where an account's index file lives.
|
||||
//
|
||||
// The standalone Next.js server in `electron/main.ts` is the SAME artifact the
|
||||
// production `Dockerfile` ships to multi-tenant deployments. An index that
|
||||
// activated unconditionally would have a shared server start writing every
|
||||
// user's mail into a server-side SQLite file. So activation is keyed on an env
|
||||
// var that ONLY `electron/main.ts` sets, and that same var supplies the path -
|
||||
// one variable doing both jobs, so they cannot drift apart.
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
|
||||
/** Set by electron/main.ts on spawn. Absent => the feature does not exist. */
|
||||
export const STORE_DIR_ENV = 'VNCMAIL_DESKTOP_STORE_DIR';
|
||||
|
||||
/**
|
||||
* The index root, or `null` when this process is not the desktop shell's
|
||||
* server. Every route must 404 on `null` - not 403, since nothing should learn
|
||||
* the routes exist in a deployment that doesn't have the feature.
|
||||
*/
|
||||
export function getStoreDir(): string | null {
|
||||
const dir = process.env[STORE_DIR_ENV]?.trim();
|
||||
if (!dir) return null;
|
||||
// Must be absolute: a relative path would resolve against the server's cwd,
|
||||
// which differs between `electron:dev` and a packaged build.
|
||||
if (!path.isAbsolute(dir)) return null;
|
||||
return dir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filenames are a hash, not `username@host`, so a directory listing is not a
|
||||
* plaintext inventory of the user's accounts. The account id itself lives only
|
||||
* inside the encrypted file (and in the renderer's own `account-registry`,
|
||||
* which already stores it in plain localStorage).
|
||||
*/
|
||||
export function accountFileToken(accountId: string): string {
|
||||
return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
export function indexDbPath(storeDir: string, accountId: string): string {
|
||||
return path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`);
|
||||
}
|
||||
|
||||
export function keyFilePath(storeDir: string, accountId: string): string {
|
||||
return path.join(storeDir, 'keys', `${accountFileToken(accountId)}.bin`);
|
||||
}
|
||||
|
||||
/** WAL siblings must be removed with the database, or a purge leaks readable pages. */
|
||||
export function dbSiblings(dbPath: string): string[] {
|
||||
return [dbPath, `${dbPath}-wal`, `${dbPath}-shm`];
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
// The index jobs.
|
||||
//
|
||||
// TWO SHAPES, both plain request-scoped work - there is no background worker,
|
||||
// no cursor, no retry ladder and no resident credential anywhere:
|
||||
//
|
||||
// 1. `indexDocuments()` - the PRIMARY path. The renderer's live JMAP push
|
||||
// connection sees a StateChange, and calls the route with the ids that
|
||||
// changed (or with no ids, meaning "refetch what's recent for this type").
|
||||
// One or a handful of objects, fetched and upserted.
|
||||
// 2. `catchUpAll()` - the FALLBACK. On app launch, backfill a bounded recent
|
||||
// window for every supported type, because anything that changed while the
|
||||
// app was closed produced no push event.
|
||||
//
|
||||
// Staleness between refreshes is acceptable by design: this is a search index
|
||||
// for a retrieval/AI feature, not a mail replica.
|
||||
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { generateAccountId } from '@/lib/account-utils';
|
||||
import { getStalwartCredentials } from '@/lib/stalwart/credentials';
|
||||
import { logger } from '@/lib/logger';
|
||||
import {
|
||||
accountHasCapability, accountIdFor, buildFilePaths, CAP_CALENDARS, CAP_CONTACTS,
|
||||
CAP_FILENODE, CAP_MAIL, fetchJmapSession, getCalendarEventsForIndex, getContactsForIndex,
|
||||
getEmailsForIndex, getFilesForIndex, hasCapability, JmapIndexError, queryCalendarEventIds,
|
||||
queryContactIds, queryFileIds, queryRecentEmailIds, type JmapSessionInfo,
|
||||
} from './jmap';
|
||||
import { extractCalendarEvent, extractContact, extractFile, extractMail } from './extract';
|
||||
import { withIndexKey } from './key';
|
||||
import { getStoreDir } from './paths';
|
||||
import { MailIndex, type ContentType, type IndexDoc } from './store';
|
||||
|
||||
/**
|
||||
* Bounded window. Small on purpose: this is the first cut of a retrieval index,
|
||||
* and a wide window turns "index on every delivery" into a slow request. The
|
||||
* event-driven path indexes single objects, so the window only bounds catch-up.
|
||||
*/
|
||||
export const INDEX_WINDOW_DAYS = 30;
|
||||
/** Calendar looks forward as well as back - upcoming events are the useful ones. */
|
||||
export const CALENDAR_FORWARD_DAYS = 180;
|
||||
/** Per-type ceiling for one catch-up pass. */
|
||||
export const CATCHUP_MAX_PER_TYPE = 500;
|
||||
/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
|
||||
export const MAX_IDS_PER_CALL = 200;
|
||||
/** Cap on body bytes requested per message from the server. */
|
||||
export const MAX_BODY_VALUE_BYTES = 256_000;
|
||||
/** Contacts and files have no useful date filter, so they are simply capped. */
|
||||
export const CONTACTS_MAX = 2_000;
|
||||
export const FILES_MAX = 2_000;
|
||||
|
||||
export interface IndexSession {
|
||||
serverUrl: string;
|
||||
authHeader: string;
|
||||
username: string;
|
||||
slot: number;
|
||||
/** `username@host` - the durable per-account key. NEVER the cookie slot. */
|
||||
accountId: string;
|
||||
}
|
||||
|
||||
export class IndexSessionError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status: number) {
|
||||
super(message);
|
||||
this.name = 'IndexSessionError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the calling request to an account and a usable Authorization header.
|
||||
*
|
||||
* Uses the SAME per-slot encrypted `jmap_stalwart_ctx` cookie that
|
||||
* `/api/settings`, `/api/push/preview` and `/api/plugin-approval-status`
|
||||
* already read (`lib/stalwart/credentials.ts`). That cookie is written by
|
||||
* `/api/auth/stalwart-context`, which the renderer syncs on every login,
|
||||
* session restore, SSO callback, account switch and token refresh
|
||||
* (`stores/auth-store.ts`, 10 call sites), and it carries a ready-made header
|
||||
* for BOTH basic and bearer accounts.
|
||||
*
|
||||
* Why this matters beyond convenience: it means the indexer never touches the
|
||||
* OAuth refresh-token cookie. A server-side refresh would rotate the token into
|
||||
* a response nobody reads while the browser kept the superseded one, and the
|
||||
* next real refresh would then fail and log the user out. Reading an
|
||||
* already-minted header cannot cause that.
|
||||
*/
|
||||
export async function resolveIndexSession(request: NextRequest): Promise<IndexSession> {
|
||||
const credentials = await getStalwartCredentials(request);
|
||||
if (!credentials) {
|
||||
throw new IndexSessionError('No JMAP auth context for this account; sign in again.', 401);
|
||||
}
|
||||
const accountId = generateAccountId(credentials.username, credentials.serverUrl);
|
||||
return { ...credentials, accountId };
|
||||
}
|
||||
|
||||
export interface IndexResult {
|
||||
accountId: string;
|
||||
/** Per-type counts of documents written. */
|
||||
written: Partial<Record<ContentType, number>>;
|
||||
/** Types the server (or this account) doesn't support, so nothing was attempted. */
|
||||
skipped: ContentType[];
|
||||
/** Non-fatal per-type failures. One broken type must not fail the whole call. */
|
||||
errors: Array<{ contentType: ContentType; message: string }>;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
function isoDaysFromNow(days: number): string {
|
||||
return new Date(Date.now() + days * 24 * 60 * 60 * 1000).toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Which types this session can actually index. Calendar/contacts are session
|
||||
* capabilities; files is a PER-ACCOUNT capability (a server can advertise
|
||||
* filenode while a specific account has it revoked - #563).
|
||||
*/
|
||||
export function supportedTypes(session: JmapSessionInfo): {
|
||||
supported: ContentType[];
|
||||
skipped: ContentType[];
|
||||
accountIds: Partial<Record<ContentType, string>>;
|
||||
} {
|
||||
const supported: ContentType[] = [];
|
||||
const skipped: ContentType[] = [];
|
||||
const accountIds: Partial<Record<ContentType, string>> = {};
|
||||
|
||||
const mailAccount = accountIdFor(session, CAP_MAIL);
|
||||
if (mailAccount) { supported.push('mail'); accountIds.mail = mailAccount; }
|
||||
else skipped.push('mail');
|
||||
|
||||
const calAccount = accountIdFor(session, CAP_CALENDARS);
|
||||
if (calAccount && hasCapability(session, CAP_CALENDARS)) {
|
||||
supported.push('calendar'); accountIds.calendar = calAccount;
|
||||
} else skipped.push('calendar');
|
||||
|
||||
const contactAccount = accountIdFor(session, CAP_CONTACTS);
|
||||
if (contactAccount && hasCapability(session, CAP_CONTACTS)) {
|
||||
supported.push('contact'); accountIds.contact = contactAccount;
|
||||
} else skipped.push('contact');
|
||||
|
||||
// Files fall back to the mail account id: Stalwart exposes FileNode on the
|
||||
// same account and does not always list a primaryAccounts entry for it.
|
||||
const fileAccount = accountIdFor(session, CAP_FILENODE) ?? mailAccount;
|
||||
if (fileAccount && accountHasCapability(session, fileAccount, CAP_FILENODE)) {
|
||||
supported.push('file'); accountIds.file = fileAccount;
|
||||
} else skipped.push('file');
|
||||
|
||||
return { supported, skipped, accountIds };
|
||||
}
|
||||
|
||||
interface FetchArgs {
|
||||
session: JmapSessionInfo;
|
||||
authHeader: string;
|
||||
jmapAccountId: string;
|
||||
ids: readonly string[] | null;
|
||||
}
|
||||
|
||||
/** Fetches and flattens one content type. `ids === null` means "the recent window". */
|
||||
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<IndexDoc[]> {
|
||||
const { session, authHeader, jmapAccountId, ids } = args;
|
||||
|
||||
switch (contentType) {
|
||||
case 'mail': {
|
||||
const targetIds = ids ?? await queryRecentEmailIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
// Chunked because bodies are big: one Email/get for 500 messages with
|
||||
// full bodies would be an enormous response.
|
||||
for (let i = 0; i < targetIds.length; i += 25) {
|
||||
const emails = await getEmailsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 25), MAX_BODY_VALUE_BYTES,
|
||||
);
|
||||
for (const email of emails) docs.push(extractMail(jmapAccountId, email));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'calendar': {
|
||||
const targetIds = ids ?? await queryCalendarEventIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||
CATCHUP_MAX_PER_TYPE,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 50) {
|
||||
const events = await getCalendarEventsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 50),
|
||||
);
|
||||
for (const event of events) docs.push(extractCalendarEvent(jmapAccountId, event));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'contact': {
|
||||
const targetIds = ids ?? await queryContactIds(session, authHeader, jmapAccountId, CONTACTS_MAX);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 100) {
|
||||
const cards = await getContactsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
|
||||
);
|
||||
for (const card of cards) docs.push(extractContact(jmapAccountId, card));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'file': {
|
||||
const targetIds = ids ?? await queryFileIds(session, authHeader, jmapAccountId, FILES_MAX);
|
||||
const nodes = [];
|
||||
for (let i = 0; i < targetIds.length; i += 100) {
|
||||
nodes.push(...await getFilesForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
|
||||
));
|
||||
}
|
||||
// Paths need the whole set in hand, so this one can't stream per chunk.
|
||||
const paths = buildFilePaths(nodes);
|
||||
return nodes
|
||||
// Directories are indexed too: "what's in the Invoices folder" is a
|
||||
// real query, and a folder row is a few bytes.
|
||||
.map((node) => extractFile(jmapAccountId, node, { path: paths.get(node.id) }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface IndexRequest {
|
||||
/** Types to touch. Empty means every supported type. */
|
||||
types?: readonly ContentType[];
|
||||
/**
|
||||
* Per-type ids to index. Omitted/empty for a type means "refetch that type's
|
||||
* recent window" (the catch-up shape).
|
||||
*/
|
||||
ids?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Per-type ids to REMOVE (a JMAP `destroyed`). */
|
||||
removed?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Drop documents outside the retention window after writing. */
|
||||
prune?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one index pass. Opens the encrypted store, fetches, upserts, closes.
|
||||
*
|
||||
* The key is fetched from the main process for the duration of this call only
|
||||
* (`withIndexKey`) and zeroed afterwards - there is no cached handle and no
|
||||
* resident key.
|
||||
*/
|
||||
export async function runIndex(
|
||||
indexSession: IndexSession,
|
||||
req: IndexRequest,
|
||||
): Promise<IndexResult> {
|
||||
const started = Date.now();
|
||||
const storeDir = getStoreDir();
|
||||
if (!storeDir) {
|
||||
throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
|
||||
}
|
||||
|
||||
const session = await fetchJmapSession(indexSession.serverUrl, indexSession.authHeader);
|
||||
|
||||
// Identity cross-check. `generateAccountId` used the username from the auth
|
||||
// context cookie; the server may canonicalise a short login (`linus`) to a
|
||||
// full address (`linus@example.com`) - which is exactly why AccountEntry
|
||||
// carries `serverIdentifiers`. Accept either form, reject anything else
|
||||
// rather than writing one account's mail into another's file.
|
||||
if (session.username) {
|
||||
const serverAccountId = generateAccountId(session.username, indexSession.serverUrl);
|
||||
if (serverAccountId !== indexSession.accountId) {
|
||||
const shortMatches = session.username.split('@')[0] === indexSession.username.split('@')[0];
|
||||
if (!shortMatches) {
|
||||
throw new IndexSessionError(
|
||||
'The JMAP session belongs to a different account than the request cookie.',
|
||||
409,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { supported, skipped, accountIds } = supportedTypes(session);
|
||||
const requested = req.types && req.types.length > 0 ? req.types : supported;
|
||||
const types = requested.filter((t) => supported.includes(t));
|
||||
const notAttempted = [...new Set([...skipped, ...requested.filter((t) => !supported.includes(t))])];
|
||||
|
||||
const written: Partial<Record<ContentType, number>> = {};
|
||||
const errors: IndexResult['errors'] = [];
|
||||
|
||||
await withIndexKey(indexSession.accountId, async (key) => {
|
||||
const index = MailIndex.open({ storeDir, accountId: indexSession.accountId, key });
|
||||
try {
|
||||
for (const contentType of types) {
|
||||
const jmapAccountId = accountIds[contentType];
|
||||
if (!jmapAccountId) continue;
|
||||
try {
|
||||
const removed = req.removed?.[contentType];
|
||||
if (removed && removed.length > 0) {
|
||||
index.remove(jmapAccountId, contentType, removed.slice(0, MAX_IDS_PER_CALL));
|
||||
}
|
||||
|
||||
const requestedIds = req.ids?.[contentType];
|
||||
const ids = requestedIds && requestedIds.length > 0
|
||||
? requestedIds.slice(0, MAX_IDS_PER_CALL)
|
||||
: null;
|
||||
|
||||
const docs = await fetchDocs(contentType, {
|
||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids,
|
||||
});
|
||||
written[contentType] = index.upsert(docs);
|
||||
|
||||
if (req.prune && contentType === 'mail') {
|
||||
// Only mail prunes by date: calendar's window looks forward,
|
||||
// contacts have no date, and file rows are metadata-sized.
|
||||
index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS));
|
||||
}
|
||||
} catch (error) {
|
||||
// One unsupported or misbehaving type must not fail the others.
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push({ contentType, message });
|
||||
if (error instanceof JmapIndexError && error.status === 401) throw error;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
index.close();
|
||||
}
|
||||
});
|
||||
|
||||
const result: IndexResult = {
|
||||
accountId: indexSession.accountId,
|
||||
written,
|
||||
skipped: notAttempted,
|
||||
errors,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
logger.info('mail-index: pass complete', {
|
||||
slot: indexSession.slot,
|
||||
written: JSON.stringify(written),
|
||||
skipped: notAttempted.join(',') || 'none',
|
||||
errors: errors.length,
|
||||
durationMs: result.durationMs,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
// The encrypted local search index: schema, open/close, upsert, search.
|
||||
//
|
||||
// One SQLite (SQLCipher) file per account. Rows are ALSO account-scoped
|
||||
// internally - `(jmap_account_id, content_type, id)` - because a single login
|
||||
// exposes the user's own JMAP account plus every delegated/shared account, and
|
||||
// JMAP ids are unique only WITHIN an account (this codebase already works
|
||||
// around that collision in `lib/jmap/client.ts:388`'s namespaceMailboxIds).
|
||||
// One file per account keeps purge trivial; the composite key keeps
|
||||
// delegated accounts from merging inside it.
|
||||
//
|
||||
// This is a SEARCH INDEX, not a mail replica. It is allowed to be stale, it is
|
||||
// allowed to be incomplete, and it can be discarded and rebuilt at any time -
|
||||
// which is why the schema-version mismatch path below simply drops everything
|
||||
// rather than migrating.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { loadSqlcipher, type SqlcipherDatabase } from './binding';
|
||||
import { dbSiblings, indexDbPath } from './paths';
|
||||
|
||||
export const SCHEMA_VERSION = 1;
|
||||
|
||||
export type ContentType = 'mail' | 'calendar' | 'contact' | 'file';
|
||||
|
||||
export const CONTENT_TYPES: readonly ContentType[] = ['mail', 'calendar', 'contact', 'file'];
|
||||
|
||||
export function isContentType(v: unknown): v is ContentType {
|
||||
return typeof v === 'string' && (CONTENT_TYPES as readonly string[]).includes(v);
|
||||
}
|
||||
|
||||
/**
|
||||
* One indexable thing, already flattened to text. Produced by the pure
|
||||
* extractors in `extract.ts` so that every JMAP-shape decision is unit-testable
|
||||
* without a database or a server.
|
||||
*/
|
||||
export interface IndexDoc {
|
||||
jmapAccountId: string;
|
||||
contentType: ContentType;
|
||||
/** JMAP id. Unique only within (jmapAccountId, contentType). */
|
||||
id: string;
|
||||
/** Subject / event title / contact display name / filename. */
|
||||
title: string;
|
||||
/** Addresses and names: sender+recipients, attendees, contact emails/phones, owner. */
|
||||
people: string;
|
||||
/** The bulk searchable text. Plain text only - never HTML. */
|
||||
body: string;
|
||||
/** ISO 8601, or null when the type has no meaningful date. Drives recency ordering. */
|
||||
occurredAt: string | null;
|
||||
/** Small type-specific extras returned verbatim to the caller (never searched). */
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface SearchHit {
|
||||
contentType: ContentType;
|
||||
id: string;
|
||||
jmapAccountId: string;
|
||||
title: string;
|
||||
people: string;
|
||||
occurredAt: string | null;
|
||||
metadata: Record<string, unknown>;
|
||||
/** FTS5 bm25 score. Lower is a better match (bm25 returns negative values). */
|
||||
score: number;
|
||||
/** Highlighted excerpt from the body, for feeding an LLM as context. */
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
const DDL = `
|
||||
CREATE TABLE IF NOT EXISTS doc (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
people TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
occurred_at TEXT,
|
||||
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||
indexed_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (jmap_account_id, content_type, id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS doc_recent
|
||||
ON doc(jmap_account_id, content_type, occurred_at DESC);
|
||||
|
||||
-- Standalone (not external-content) FTS5: the text is duplicated into this
|
||||
-- table and kept in step manually on upsert. External content would avoid the
|
||||
-- duplication but requires deleting the old FTS row using its OLD column
|
||||
-- values, which an upsert does not have to hand - a well-known source of
|
||||
-- silently-stale FTS rows. At this scale (a bounded recent window) the
|
||||
-- duplication is the cheaper correctness trade.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5(
|
||||
title, people, body,
|
||||
tokenize='unicode61 remove_diacritics 2'
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);
|
||||
`;
|
||||
|
||||
export class MailIndexUnavailableError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'MailIndexUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that the file we just opened is REALLY encrypted.
|
||||
*
|
||||
* This is not defensive boilerplate, it guards the sharpest landmine found
|
||||
* while designing this: on both `node:sqlite` and plain `better-sqlite3`,
|
||||
* `PRAGMA key = ...` is **silently accepted and does nothing** - no error, a
|
||||
* working database, and the mail sitting on disk in cleartext. Verified by
|
||||
* writing a file and recovering a canary string from the raw bytes.
|
||||
*
|
||||
* The check is on the VALUE, not the row count: a non-cipher binding returns
|
||||
* ZERO ROWS for `PRAGMA cipher_version`, so a naive `!== ''` comparison over a
|
||||
* missing row passes vacuously. Require a non-empty string.
|
||||
*/
|
||||
function assertEncrypted(db: SqlcipherDatabase, dbPath: string): void {
|
||||
const rows = db.pragma('cipher_version');
|
||||
const value =
|
||||
Array.isArray(rows) && rows.length > 0 && rows[0] && typeof rows[0] === 'object'
|
||||
? (rows[0] as Record<string, unknown>).cipher_version
|
||||
: undefined;
|
||||
if (typeof value !== 'string' || value.trim().length === 0) {
|
||||
db.close();
|
||||
throw new MailIndexUnavailableError(
|
||||
`Refusing to use ${path.basename(dbPath)}: the SQLite binding reports no SQLCipher ` +
|
||||
`support (PRAGMA cipher_version returned ${JSON.stringify(rows)}), so the index ` +
|
||||
`would be written in cleartext.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface OpenOptions {
|
||||
storeDir: string;
|
||||
accountId: string;
|
||||
/** Raw 32-byte key. Used as SQLCipher's raw key (no KDF) via `PRAGMA key = "x'..'"`. */
|
||||
key: Buffer;
|
||||
}
|
||||
|
||||
export class MailIndex {
|
||||
private constructor(
|
||||
private readonly db: SqlcipherDatabase,
|
||||
readonly dbPath: string,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Opens (creating if needed) the account's index. Throws
|
||||
* MailIndexUnavailableError when the native binding is absent or the file is
|
||||
* not actually encrypted; the caller turns the feature off rather than
|
||||
* falling back to something unencrypted.
|
||||
*/
|
||||
static open({ storeDir, accountId, key }: OpenOptions): MailIndex {
|
||||
const Database = loadSqlcipher();
|
||||
if (!Database) {
|
||||
throw new MailIndexUnavailableError(
|
||||
'@signalapp/sqlcipher is not installed for this platform (it is an optional dependency).',
|
||||
);
|
||||
}
|
||||
if (key.length !== 32) {
|
||||
throw new MailIndexUnavailableError(`Index key must be 32 bytes, got ${key.length}.`);
|
||||
}
|
||||
|
||||
const dbPath = indexDbPath(storeDir, accountId);
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: 0o700 });
|
||||
|
||||
let db = new Database(dbPath);
|
||||
// The key pragma must be the FIRST statement on the connection. Hex form
|
||||
// means SQLCipher uses these 32 bytes as the raw key with no KDF, which is
|
||||
// right for a random key (a passphrase would want the KDF).
|
||||
db.pragma(`key = "x'${key.toString('hex')}'"`);
|
||||
assertEncrypted(db, dbPath);
|
||||
|
||||
// A wrong key surfaces here rather than at open: SQLCipher only reads the
|
||||
// header lazily. Treat it as "unreadable" and rebuild from scratch - the
|
||||
// index is derived data, so there is nothing to recover and never anything
|
||||
// to prompt the user for (the key was never a user secret).
|
||||
let version: number | null;
|
||||
try {
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
version = readSchemaVersion(db);
|
||||
} catch {
|
||||
db.close();
|
||||
for (const f of dbSiblings(dbPath)) {
|
||||
try { fs.rmSync(f, { force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
db = new Database(dbPath);
|
||||
db.pragma(`key = "x'${key.toString('hex')}'"`);
|
||||
assertEncrypted(db, dbPath);
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
version = null;
|
||||
}
|
||||
|
||||
if (version !== null && version !== SCHEMA_VERSION) {
|
||||
// Rebuildable derived data: drop, don't migrate.
|
||||
db.exec('DROP TABLE IF EXISTS doc_fts; DROP TABLE IF EXISTS doc; DROP TABLE IF EXISTS meta;');
|
||||
version = null;
|
||||
}
|
||||
if (version === null) {
|
||||
db.exec(DDL);
|
||||
db.prepare('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)').run([
|
||||
'schema_version',
|
||||
String(SCHEMA_VERSION),
|
||||
]);
|
||||
}
|
||||
|
||||
return new MailIndex(db, dbPath);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.db.close(); } catch { /* already closed */ }
|
||||
}
|
||||
|
||||
/**
|
||||
* Upserts documents and keeps the FTS rows in step. Returns the number of
|
||||
* rows written. One transaction for the whole batch - a partially-applied
|
||||
* batch is harmless (it is an index) but a transaction is faster.
|
||||
*/
|
||||
upsert(docs: readonly IndexDoc[]): number {
|
||||
if (docs.length === 0) return 0;
|
||||
|
||||
const upsertDoc = this.db.prepare(`
|
||||
INSERT INTO doc (jmap_account_id, content_type, id, title, people, body,
|
||||
occurred_at, metadata_json, indexed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(jmap_account_id, content_type, id) DO UPDATE SET
|
||||
title = excluded.title, people = excluded.people, body = excluded.body,
|
||||
occurred_at = excluded.occurred_at, metadata_json = excluded.metadata_json,
|
||||
indexed_at = excluded.indexed_at
|
||||
RETURNING rowid
|
||||
`);
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const insertFts = this.db.prepare(
|
||||
'INSERT INTO doc_fts (rowid, title, people, body) VALUES (?, ?, ?, ?)',
|
||||
);
|
||||
|
||||
const now = Date.now();
|
||||
let written = 0;
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const d of docs) {
|
||||
const row = upsertDoc.get([
|
||||
d.jmapAccountId, d.contentType, d.id,
|
||||
d.title, d.people, d.body,
|
||||
d.occurredAt, JSON.stringify(d.metadata ?? {}), now,
|
||||
]);
|
||||
const rowid = row?.rowid;
|
||||
if (typeof rowid !== 'number') continue;
|
||||
// ON CONFLICT preserves the rowid, so delete-then-insert replaces the
|
||||
// old FTS row rather than accumulating duplicates for one document.
|
||||
deleteFts.run([rowid]);
|
||||
insertFts.run([rowid, d.title, d.people, d.body]);
|
||||
written++;
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
/** Removes documents by id (a JMAP `destroyed` id, or a stale row). */
|
||||
remove(jmapAccountId: string, contentType: ContentType, ids: readonly string[]): number {
|
||||
if (ids.length === 0) return 0;
|
||||
const findRow = this.db.prepare(
|
||||
'SELECT rowid FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
|
||||
);
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const deleteDoc = this.db.prepare(
|
||||
'DELETE FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
|
||||
);
|
||||
let removed = 0;
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const row = findRow.get([jmapAccountId, contentType, id]);
|
||||
if (typeof row?.rowid === 'number') deleteFts.run([row.rowid]);
|
||||
removed += deleteDoc.run([jmapAccountId, contentType, id]).changes;
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-text search - the retrieval surface an AI feature calls to gather
|
||||
* context. `types` empty/omitted searches everything.
|
||||
*/
|
||||
search(opts: {
|
||||
query: string;
|
||||
types?: readonly ContentType[];
|
||||
limit?: number;
|
||||
snippetTokens?: number;
|
||||
}): SearchHit[] {
|
||||
const match = toFtsMatchQuery(opts.query);
|
||||
if (!match) return [];
|
||||
|
||||
const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200);
|
||||
const tokens = Math.min(Math.max(opts.snippetTokens ?? 24, 4), 64);
|
||||
const types = opts.types && opts.types.length > 0 ? opts.types : null;
|
||||
const typeFilter = types ? ` AND d.content_type IN (${types.map(() => '?').join(',')})` : '';
|
||||
|
||||
// bm25 weights: a hit in the title or in a name/address is a stronger
|
||||
// signal than one in a long body, and for RAG the title is what makes a
|
||||
// retrieved chunk recognisable.
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people,
|
||||
d.occurred_at, d.metadata_json,
|
||||
bm25(doc_fts, 8.0, 4.0, 1.0) AS score,
|
||||
snippet(doc_fts, 2, '[', ']', '…', ${tokens}) AS snip
|
||||
FROM doc_fts
|
||||
JOIN doc d ON d.rowid = doc_fts.rowid
|
||||
WHERE doc_fts MATCH ?${typeFilter}
|
||||
ORDER BY score ASC, d.occurred_at DESC
|
||||
LIMIT ?
|
||||
`)
|
||||
.all([match, ...(types ?? []), limit]);
|
||||
|
||||
return rows.map((r) => ({
|
||||
contentType: String(r.content_type) as ContentType,
|
||||
id: String(r.id),
|
||||
jmapAccountId: String(r.jmap_account_id),
|
||||
title: String(r.title ?? ''),
|
||||
people: String(r.people ?? ''),
|
||||
occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at),
|
||||
metadata: safeParseObject(r.metadata_json),
|
||||
score: typeof r.score === 'number' ? r.score : 0,
|
||||
snippet: String(r.snip ?? ''),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Per-type counts and freshness, for the Settings UI and for debugging. */
|
||||
stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT content_type, COUNT(*) AS n, MAX(occurred_at) AS newest, MAX(indexed_at) AS indexed
|
||||
FROM doc GROUP BY content_type ORDER BY content_type
|
||||
`)
|
||||
.all()
|
||||
.map((r) => ({
|
||||
contentType: String(r.content_type),
|
||||
count: Number(r.n ?? 0),
|
||||
newest: r.newest === null || r.newest === undefined ? null : String(r.newest),
|
||||
indexedAt: typeof r.indexed === 'number' ? r.indexed : null,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Ids already present, so a catch-up pass can skip re-fetching bodies. */
|
||||
existingIds(jmapAccountId: string, contentType: ContentType): Set<string> {
|
||||
const rows = this.db
|
||||
.prepare('SELECT id FROM doc WHERE jmap_account_id = ? AND content_type = ?')
|
||||
.all([jmapAccountId, contentType]);
|
||||
return new Set(rows.map((r) => String(r.id)));
|
||||
}
|
||||
|
||||
/** Drops documents older than the retention floor for a type. */
|
||||
pruneOlderThan(jmapAccountId: string, contentType: ContentType, isoFloor: string): number {
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT rowid FROM doc
|
||||
WHERE jmap_account_id = ? AND content_type = ?
|
||||
AND occurred_at IS NOT NULL AND occurred_at < ?
|
||||
`)
|
||||
.all([jmapAccountId, contentType, isoFloor]);
|
||||
if (rows.length === 0) return 0;
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const deleteDoc = this.db.prepare('DELETE FROM doc WHERE rowid = ?');
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const r of rows) {
|
||||
deleteFts.run([r.rowid]);
|
||||
deleteDoc.run([r.rowid]);
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
}
|
||||
|
||||
function readSchemaVersion(db: SqlcipherDatabase): number | null {
|
||||
try {
|
||||
const row = db.prepare("SELECT v FROM meta WHERE k = 'schema_version'").get();
|
||||
if (!row || row.v === undefined) return null;
|
||||
const n = Number(row.v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
} catch {
|
||||
// `meta` doesn't exist yet - a fresh file.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function safeParseObject(v: unknown): Record<string, unknown> {
|
||||
if (typeof v !== 'string') return {};
|
||||
try {
|
||||
const parsed = JSON.parse(v);
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns arbitrary user text into a safe FTS5 MATCH expression.
|
||||
*
|
||||
* FTS5's query syntax is not SQL, so parameter binding does NOT protect it: a
|
||||
* bare `"` or a stray `*`/`NEAR`/`:` in user input raises
|
||||
* `fts5: syntax error`, which would turn a normal search box into a 500. Every
|
||||
* token is quoted (making it a literal phrase) and a trailing `*` is added to
|
||||
* the last token so typing continues to match as the user types.
|
||||
*
|
||||
* Exported for unit testing - it is the one piece of this file with no
|
||||
* database dependency and the most ways to be wrong.
|
||||
*/
|
||||
export function toFtsMatchQuery(raw: string): string | null {
|
||||
if (typeof raw !== 'string') return null;
|
||||
// Split on anything that isn't a word character or an intra-word mark. Keeps
|
||||
// unicode letters (so "Müller" and "東京" survive) via the u flag.
|
||||
const tokens = raw
|
||||
.normalize('NFC')
|
||||
.split(/[^\p{L}\p{N}_@.'-]+/u)
|
||||
.map((t) => t.replace(/^['-]+|['-]+$/g, ''))
|
||||
.filter((t) => t.length > 0)
|
||||
.slice(0, 24);
|
||||
if (tokens.length === 0) return null;
|
||||
return tokens
|
||||
.map((t, i) => {
|
||||
const quoted = `"${t.replace(/"/g, '""')}"`;
|
||||
// Prefix-match only the final token, and only if it's long enough to not
|
||||
// match half the mailbox.
|
||||
return i === tokens.length - 1 && t.length >= 3 ? `${quoted}*` : quoted;
|
||||
})
|
||||
.join(' AND ');
|
||||
}
|
||||
Reference in New Issue
Block a user