feat(mail-index): encrypted SQLite/FTS5 index over mail, calendar, contacts, files
An on-device, SQLCipher-encrypted full-text index the app can retrieve from to
feed an LLM ("prompt against"), for the Electron desktop shell only.
Shape: no persistent background worker and no resident credential. Indexing is
a normal request-scoped API route, triggered by the renderer's EXISTING live
JMAP push connection - so it reacts to each delivery/change rather than polling.
- lib/mail-index/binding.ts guarded require of the optional native binding
- lib/mail-index/paths.ts the VNCMAIL_DESKTOP_STORE_DIR gate + hashed paths
- lib/mail-index/store.ts schema, upsert, FTS5 search, encryption assertion
- lib/mail-index/extract.ts PURE JMAP-object -> document extractors
- lib/mail-index/jmap.ts minimal stateless server-side JMAP client
- lib/mail-index/key.ts per-job key fetch over the inherited fd
- lib/mail-index/reindex.ts the job + slot->account resolution
- electron/key-service.ts safeStorage wrap/unwrap, served over fd 3
- app/api/offline/reindex POST, event-driven + catch-up
- app/api/offline/search GET, the retrieval surface (hits + contextBlock)
- lib/mail-index-client.ts renderer client; StateChange -> index call
- components/settings/local-index-settings.tsx status + manual catch-up
Decisions worth knowing:
* `@signalapp/sqlcipher` is an OPTIONAL dependency with a guarded runtime
require. It publishes six N-API prebuilds and NO build sources, and both
Dockerfiles are node:24-alpine (musl, no matching prebuild) - as a hard
dependency it would break the production image and the integration fixture's
webmail container, neither of which wants this feature.
* Credentials come from the existing per-slot encrypted `jmap_stalwart_ctx`
cookie via lib/stalwart/credentials.ts - the same helper /api/settings and
/api/push/preview already use. It carries a ready-made header for basic AND
bearer accounts, so the indexer never touches the OAuth refresh-token cookie;
a server-side refresh would rotate a token into a response nobody reads and
silently log the user out.
* The encryption key crosses main -> server over an INHERITED FILE DESCRIPTOR,
never an environment variable: env is readable by any process running as the
same OS user, which would defeat using the OS keychain at all. Fetched per
job and zeroed after, so there is no long-lived key copy.
* safeStorage's Linux `basic_text` backend (no keyring) is treated as refusal,
not degradation - it "encrypts" with a hardcoded public password, which would
look like an encrypted mailbox while providing nothing.
getSelectedStorageBackend() is Linux-only and platform-guarded.
* Every store open asserts `PRAGMA cipher_version` returns a non-empty STRING,
not merely a row: a non-cipher binding returns ZERO ROWS, so a row-count check
would pass vacuously while writing the mailbox to disk in cleartext.
* Files are indexed by name/path/date/size only - NOT by extracted content.
Text extraction from arbitrary PDFs/office documents is a separate problem.
* Account-scoped composite keys `(jmap_account_id, content_type, id)` are kept
even though there is one file per account: one login exposes delegated/shared
JMAP accounts too, and JMAP ids are unique only within an account.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Bernd Rodler
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
16466c7296
commit
b966d285a9
@@ -0,0 +1,199 @@
|
||||
// Renderer-side client for the encrypted local search index.
|
||||
//
|
||||
// The index is EVENT-DRIVEN: the renderer already holds the live JMAP push
|
||||
// connection (WebSocket -> SSE -> polling, `lib/jmap/client.ts`'s
|
||||
// setupPushNotifications), so the moment a StateChange announces new mail, a
|
||||
// calendar change, a contact edit or a file upload, this posts to the reindex
|
||||
// route. No polling loop, no background worker, no long-lived credential -
|
||||
// just one more authenticated fetch from the place the push already arrives.
|
||||
//
|
||||
// Every function here is best-effort and never throws: a search index failing
|
||||
// to update must never break the mail UI.
|
||||
|
||||
import { apiFetch } from '@/lib/browser-navigation';
|
||||
import { debug } from '@/lib/debug';
|
||||
import type { StateChange } from '@/lib/jmap/types';
|
||||
|
||||
export type IndexContentType = 'mail' | 'calendar' | 'contact' | 'file';
|
||||
|
||||
export interface IndexRunResult {
|
||||
ok: boolean;
|
||||
written?: Partial<Record<IndexContentType, number>>;
|
||||
skipped?: IndexContentType[];
|
||||
errors?: Array<{ contentType: IndexContentType; message: string }>;
|
||||
durationMs?: number;
|
||||
/** Set when the feature isn't available (not the desktop shell, no keyring, no binding). */
|
||||
unavailable?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps JMAP `StateChange` type keys onto our content types.
|
||||
*
|
||||
* The transport is already type-generic - the WebSocket handler
|
||||
* (`client.ts:6308-6316`) and the SSE handler (`:6505`) pass the whole
|
||||
* `changed` map through untouched, and the WS subscribes with
|
||||
* `dataTypes: null` (every type) - so anything the server pushes arrives here.
|
||||
*
|
||||
* `Mailbox` is deliberately NOT mapped: a Mailbox state change is usually just
|
||||
* an unread-count move, and it fires constantly. `Email` covers the cases that
|
||||
* change indexable content.
|
||||
*/
|
||||
const STATE_TYPE_TO_CONTENT: Record<string, IndexContentType> = {
|
||||
Email: 'mail',
|
||||
Calendar: 'calendar',
|
||||
CalendarEvent: 'calendar',
|
||||
ContactCard: 'contact',
|
||||
AddressBook: 'contact',
|
||||
FileNode: 'file',
|
||||
};
|
||||
|
||||
export function contentTypesFromStateChange(change: StateChange): IndexContentType[] {
|
||||
const out = new Set<IndexContentType>();
|
||||
for (const perAccount of Object.values(change.changed ?? {})) {
|
||||
for (const stateType of Object.keys(perAccount ?? {})) {
|
||||
const mapped = STATE_TYPE_TO_CONTENT[stateType];
|
||||
if (mapped) out.add(mapped);
|
||||
}
|
||||
}
|
||||
return [...out];
|
||||
}
|
||||
|
||||
export interface IndexRequestOptions {
|
||||
types?: readonly IndexContentType[];
|
||||
/**
|
||||
* Per-type ids to index. Supply them whenever the renderer already knows
|
||||
* which objects changed - it turns the call into a couple of `Foo/get`s
|
||||
* instead of a windowed query. Mail is the frequent case and the one where
|
||||
* this matters.
|
||||
*/
|
||||
ids?: Partial<Record<IndexContentType, string[]>>;
|
||||
/** Backfill the recent window for every supported type, and prune. */
|
||||
catchUp?: boolean;
|
||||
/** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */
|
||||
slot?: number;
|
||||
}
|
||||
|
||||
let inFlight: Promise<IndexRunResult> | null = null;
|
||||
/** Set once the server says the feature isn't there, so we stop asking. */
|
||||
let knownUnavailable = false;
|
||||
|
||||
/**
|
||||
* Posts one index request. Single-flighted: a burst of deliveries coalesces
|
||||
* into the in-flight call rather than queueing N overlapping SQLite writers.
|
||||
*/
|
||||
export async function requestIndex(options: IndexRequestOptions = {}): Promise<IndexRunResult> {
|
||||
if (knownUnavailable) return { ok: false, unavailable: true };
|
||||
if (inFlight) return inFlight;
|
||||
|
||||
const query = typeof options.slot === 'number' ? `?slot=${options.slot}` : '';
|
||||
const run = (async (): Promise<IndexRunResult> => {
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/reindex${query}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
types: options.types,
|
||||
ids: options.ids,
|
||||
catchUp: options.catchUp === true,
|
||||
}),
|
||||
});
|
||||
|
||||
// 404 = not the desktop shell (or the feature is gated off). Permanent for
|
||||
// this page load; stop asking so a busy mailbox doesn't post per delivery.
|
||||
if (response.status === 404) {
|
||||
knownUnavailable = true;
|
||||
return { ok: false, unavailable: true };
|
||||
}
|
||||
if (response.status === 503) {
|
||||
// No keyring / no native binding / no key channel. Also permanent for
|
||||
// this session, and the message is worth surfacing in Settings.
|
||||
knownUnavailable = true;
|
||||
const body = await response.json().catch(() => ({}));
|
||||
return { ok: false, unavailable: true, error: body?.error };
|
||||
}
|
||||
if (!response.ok) {
|
||||
const body = await response.json().catch(() => ({}));
|
||||
return { ok: false, error: body?.error || `HTTP ${response.status}` };
|
||||
}
|
||||
const body = await response.json();
|
||||
debug.log('push', '[index] reindex done', body?.written, body?.errors);
|
||||
return {
|
||||
ok: true,
|
||||
written: body?.written,
|
||||
skipped: body?.skipped,
|
||||
errors: body?.errors,
|
||||
durationMs: body?.durationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
||||
} finally {
|
||||
inFlight = null;
|
||||
}
|
||||
})();
|
||||
|
||||
inFlight = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
/**
|
||||
* The event-driven entry point, called from the push handler.
|
||||
*
|
||||
* `mailIds` lets the caller hand over the ids it already has (the refreshed
|
||||
* mailbox page), so the frequent mail case costs one `Email/get` rather than a
|
||||
* 30-day query. The other three types are rare events (a contact edit, a file
|
||||
* upload, a calendar change), so they fall back to their own bounded queries.
|
||||
*/
|
||||
export function indexOnStateChange(
|
||||
change: StateChange,
|
||||
opts: { mailIds?: string[]; slot?: number } = {},
|
||||
): void {
|
||||
if (knownUnavailable) return;
|
||||
const types = contentTypesFromStateChange(change);
|
||||
if (types.length === 0) return;
|
||||
|
||||
const ids: Partial<Record<IndexContentType, string[]>> = {};
|
||||
if (types.includes('mail') && opts.mailIds && opts.mailIds.length > 0) {
|
||||
ids.mail = opts.mailIds.slice(0, 100);
|
||||
}
|
||||
|
||||
// Fire-and-forget on purpose: this runs inside the push handler, and the mail
|
||||
// UI must not wait on a search index.
|
||||
void requestIndex({ types, ids: Object.keys(ids).length > 0 ? ids : undefined, slot: opts.slot });
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch-time catch-up: backfills whatever changed while the app was closed,
|
||||
* for which no push event was ever delivered. Also the recovery path for the
|
||||
* polling transport, which has no signal for contacts or files at all
|
||||
* (`client.ts`'s buildStatePollingRequest polls Mailbox/Email/Calendar/
|
||||
* CalendarEvent/SieveScript only).
|
||||
*/
|
||||
export async function catchUpIndex(slot?: number): Promise<IndexRunResult> {
|
||||
return requestIndex({ catchUp: true, slot });
|
||||
}
|
||||
|
||||
export interface IndexStats {
|
||||
contentType: string;
|
||||
count: number;
|
||||
newest: string | null;
|
||||
indexedAt: number | null;
|
||||
}
|
||||
|
||||
/** Reads per-type counts without searching. Used by the Settings panel. */
|
||||
export async function fetchIndexStats(slot?: number): Promise<IndexStats[] | null> {
|
||||
const slotQuery = typeof slot === 'number' ? `&slot=${slot}` : '';
|
||||
try {
|
||||
const response = await apiFetch(`/api/offline/search?stats=true${slotQuery}`);
|
||||
if (!response.ok) return null;
|
||||
const body = await response.json();
|
||||
return Array.isArray(body?.stats) ? (body.stats as IndexStats[]) : [];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resets the "don't ask again" latch - e.g. after the user signs in again. */
|
||||
export function resetIndexAvailability(): void {
|
||||
knownUnavailable = false;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Guarded loader for the SQLCipher native binding.
|
||||
//
|
||||
// WHY THIS FILE EXISTS AT ALL: `@signalapp/sqlcipher` is declared in
|
||||
// package.json's `optionalDependencies`, not `dependencies`, and it MUST stay
|
||||
// there. It publishes six N-API prebuilds (darwin/linux/win32 x arm64/x64) and
|
||||
// **no build sources at all** - the published tarball has no `binding.gyp`, no
|
||||
// `src/`, no `deps/`. Its install script is `node-gyp-build`, which falls back
|
||||
// to `node-gyp rebuild` when no prebuild matches, and that fallback cannot
|
||||
// succeed without sources. So on a platform with no matching prebuild the
|
||||
// install FAILS.
|
||||
//
|
||||
// Both Dockerfiles in this repo are `FROM node:24-alpine` + `npm ci`
|
||||
// (`Dockerfile:1-4`, `integration/webmail.Dockerfile:15-19`). Alpine is musl;
|
||||
// there is no `linuxmusl-*` prebuild (and the glibc prebuild could not load
|
||||
// there anyway). As a hard `dependencies` entry this would break the
|
||||
// production image build and the integration fixture's webmail container -
|
||||
// neither of which wants this feature, they just need `npm ci` to exit 0.
|
||||
// `optionalDependencies` makes npm treat that install failure as non-fatal and
|
||||
// simply omit the package.
|
||||
//
|
||||
// The cost of that choice is exactly this module: the require must be guarded
|
||||
// at runtime, because "installed" is no longer guaranteed. Callers get
|
||||
// `null` and the feature turns itself off, which is the correct behaviour for
|
||||
// a desktop-only search index in a server that may not be a desktop.
|
||||
|
||||
/**
|
||||
* Minimal structural type for the bits of `@signalapp/sqlcipher` we use.
|
||||
*
|
||||
* Deliberately hand-written rather than `typeof import('@signalapp/sqlcipher')`:
|
||||
* the package is optional, so a type-only import would make `tsc` fail on any
|
||||
* machine where the install was skipped - which is every Alpine CI container.
|
||||
*
|
||||
* NOTE the parameter shape. `@signalapp/sqlcipher` is NOT drop-in compatible
|
||||
* with better-sqlite3 here: its `#checkParams` throws
|
||||
* `TypeError: Params must be either object or array`, so `stmt.run(a, b, c)`
|
||||
* (varargs, which better-sqlite3 accepts) is a runtime error. Always pass a
|
||||
* single array or object. Found by executing it, not by reading the types.
|
||||
*/
|
||||
export interface SqlcipherStatement {
|
||||
run(params?: readonly unknown[] | Record<string, unknown>): { changes: number; lastInsertRowid: number };
|
||||
get(params?: readonly unknown[] | Record<string, unknown>): Record<string, unknown> | undefined;
|
||||
all(params?: readonly unknown[] | Record<string, unknown>): Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface SqlcipherDatabase {
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): SqlcipherStatement;
|
||||
pragma(source: string): unknown;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface SqlcipherConstructor {
|
||||
new (path?: string): SqlcipherDatabase;
|
||||
}
|
||||
|
||||
let cached: SqlcipherConstructor | null | undefined;
|
||||
|
||||
/**
|
||||
* Returns the Database constructor, or `null` when the optional native binding
|
||||
* is not installed / cannot load on this platform. Never throws.
|
||||
*
|
||||
* Memoised on both outcomes so a missing binding costs one failed require per
|
||||
* process rather than one per request.
|
||||
*/
|
||||
export function loadSqlcipher(): SqlcipherConstructor | null {
|
||||
if (cached !== undefined) return cached;
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const mod = require('@signalapp/sqlcipher') as
|
||||
| { default?: SqlcipherConstructor }
|
||||
| SqlcipherConstructor;
|
||||
const ctor = (mod as { default?: SqlcipherConstructor }).default ?? (mod as SqlcipherConstructor);
|
||||
cached = typeof ctor === 'function' ? ctor : null;
|
||||
} catch {
|
||||
cached = null;
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
/** True when the local index can work at all in this process. */
|
||||
export function isSqlcipherAvailable(): boolean {
|
||||
return loadSqlcipher() !== null;
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
// PURE JMAP-object -> IndexDoc extractors.
|
||||
//
|
||||
// Deliberately free of database, network and store access so every shape
|
||||
// decision here is unit-testable on its own. The JMAP shapes are awkward
|
||||
// enough (JSContact keyed maps, JSCalendar participants, FileNode's `modified`
|
||||
// rather than `updated`) that this is where the bugs would otherwise hide.
|
||||
|
||||
import type { Email, CalendarEvent, ContactCard, FileNode, EmailAddress } from '@/lib/jmap/types';
|
||||
import type { IndexDoc } from './store';
|
||||
|
||||
/** Hard cap on indexed body text per document. Keeps one enormous mail from dominating the file. */
|
||||
export const MAX_BODY_CHARS = 32_000;
|
||||
|
||||
/**
|
||||
* Minimal HTML -> text, for mail that has no `text/plain` alternative.
|
||||
*
|
||||
* Not a sanitiser and not trying to be: this output is never rendered, only
|
||||
* tokenised by FTS5 and possibly handed to an LLM as context. The repo's
|
||||
* `dompurify` needs a DOM and this runs in Node, so a DOM-free reduction is the
|
||||
* right tool. Order matters - script/style content must go before tags are
|
||||
* stripped, or their contents would leak into the index as searchable text.
|
||||
*/
|
||||
export function htmlToText(html: string): string {
|
||||
return html
|
||||
.replace(/<!--[\s\S]*?-->/g, ' ')
|
||||
.replace(/<(script|style|head)\b[\s\S]*?<\/\1>/gi, ' ')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/(p|div|tr|li|h[1-6]|blockquote)>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/&/gi, '&')
|
||||
.replace(/</gi, '<')
|
||||
.replace(/>/gi, '>')
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/&#(\d+);/g, (_m, d: string) => {
|
||||
const code = Number(d);
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
|
||||
})
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_m, h: string) => {
|
||||
const code = parseInt(h, 16);
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' ';
|
||||
})
|
||||
.replace(/[ \t\u00a0]+/g, ' ')
|
||||
.replace(/\s*\n\s*/g, '\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function normaliseText(s: string | null | undefined): string {
|
||||
if (!s) return '';
|
||||
return s.replace(/\r\n?/g, '\n').replace(/[ \t\u00a0]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
function clamp(s: string, max = MAX_BODY_CHARS): string {
|
||||
return s.length <= max ? s : s.slice(0, max);
|
||||
}
|
||||
|
||||
function formatAddresses(list: readonly EmailAddress[] | undefined): string {
|
||||
if (!list || list.length === 0) return '';
|
||||
return list
|
||||
.map((a) => [a.name, a.email].filter((p) => typeof p === 'string' && p.length > 0).join(' '))
|
||||
.filter((s) => s.length > 0)
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
/** Values of a JSContact/JSCalendar keyed map, in a stable order. */
|
||||
function mapValues<T>(m: Record<string, T> | null | undefined): T[] {
|
||||
if (!m || typeof m !== 'object') return [];
|
||||
return Object.keys(m).sort().map((k) => m[k]);
|
||||
}
|
||||
|
||||
function joinUnique(parts: Array<string | undefined | null>): string {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const p of parts) {
|
||||
const v = typeof p === 'string' ? p.trim() : '';
|
||||
if (!v || seen.has(v)) continue;
|
||||
seen.add(v);
|
||||
out.push(v);
|
||||
}
|
||||
return out.join(', ');
|
||||
}
|
||||
|
||||
// ── mail ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Resolves an Email's plain-text body from `bodyValues`, preferring the
|
||||
* `text/plain` alternative and falling back to flattening the HTML one.
|
||||
*
|
||||
* `textBody`/`htmlBody` reference parts by `partId`; the text itself only
|
||||
* arrives in `bodyValues` when the `Email/get` asked for it
|
||||
* (`fetchTextBodyValues` / `fetchHTMLBodyValues`). A caller that forgets that
|
||||
* gets an empty body rather than an error, which is exactly the kind of silent
|
||||
* hole worth naming here.
|
||||
*/
|
||||
export function emailBodyText(email: Email): string {
|
||||
const values = email.bodyValues ?? {};
|
||||
const fromParts = (parts: typeof email.textBody): string =>
|
||||
(parts ?? [])
|
||||
.map((p) => values[p.partId]?.value ?? '')
|
||||
.filter((v) => v.length > 0)
|
||||
.join('\n\n');
|
||||
|
||||
const plain = fromParts(email.textBody);
|
||||
if (plain.trim().length > 0) return normaliseText(plain);
|
||||
|
||||
const html = fromParts(email.htmlBody);
|
||||
if (html.trim().length > 0) return normaliseText(htmlToText(html));
|
||||
|
||||
// Last resort: the server-computed preview. Better than nothing for a search
|
||||
// index, and it costs no extra round trip.
|
||||
return normaliseText(email.preview);
|
||||
}
|
||||
|
||||
export function extractMail(jmapAccountId: string, email: Email): IndexDoc {
|
||||
const body = clamp(emailBodyText(email));
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'mail',
|
||||
id: email.id,
|
||||
title: normaliseText(email.subject) || '(no subject)',
|
||||
people: joinUnique([
|
||||
formatAddresses(email.from),
|
||||
formatAddresses(email.to),
|
||||
formatAddresses(email.cc),
|
||||
]),
|
||||
body,
|
||||
occurredAt: email.receivedAt ?? null,
|
||||
metadata: {
|
||||
threadId: email.threadId,
|
||||
from: email.from?.[0]?.email ?? null,
|
||||
fromName: email.from?.[0]?.name ?? null,
|
||||
hasAttachment: !!email.hasAttachment,
|
||||
size: email.size ?? null,
|
||||
mailboxIds: Object.keys(email.mailboxIds ?? {}),
|
||||
preview: normaliseText(email.preview).slice(0, 300),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── calendar ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function extractCalendarEvent(jmapAccountId: string, event: CalendarEvent): IndexDoc {
|
||||
const participants = mapValues(event.participants);
|
||||
const participantText = joinUnique(
|
||||
participants.flatMap((p) => [
|
||||
p?.name,
|
||||
p?.email,
|
||||
p?.calendarAddress?.replace(/^mailto:/i, ''),
|
||||
...Object.values(p?.sendTo ?? {}).map((v) =>
|
||||
typeof v === 'string' ? v.replace(/^mailto:/i, '') : '',
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
const locations = mapValues(event.locations)
|
||||
.map((l) => normaliseText(l?.name))
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
// `descriptionContentType` can legitimately be text/html.
|
||||
const rawDescription = normaliseText(event.description);
|
||||
const description = /html/i.test(event.descriptionContentType ?? '')
|
||||
? normaliseText(htmlToText(rawDescription))
|
||||
: rawDescription;
|
||||
|
||||
const keywords = Object.keys(event.keywords ?? {});
|
||||
const categories = Object.keys(event.categories ?? {});
|
||||
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'calendar',
|
||||
id: event.id,
|
||||
title: normaliseText(event.title) || '(untitled event)',
|
||||
people: joinUnique([event.organizerCalendarAddress?.replace(/^mailto:/i, ''), participantText]),
|
||||
body: clamp(
|
||||
[description, locations.join(', '), keywords.join(' '), categories.join(' ')]
|
||||
.filter((s) => s.length > 0)
|
||||
.join('\n\n'),
|
||||
),
|
||||
// `utcStart` is the resolved instant the app computes; `start` is local
|
||||
// wall-clock without a zone, so prefer utcStart for ordering.
|
||||
occurredAt: event.utcStart ?? event.start ?? null,
|
||||
metadata: {
|
||||
start: event.start ?? null,
|
||||
utcStart: event.utcStart ?? null,
|
||||
utcEnd: event.utcEnd ?? null,
|
||||
timeZone: event.timeZone ?? null,
|
||||
showWithoutTime: !!event.showWithoutTime,
|
||||
status: event.status ?? null,
|
||||
locations,
|
||||
calendarIds: Object.keys(event.calendarIds ?? {}),
|
||||
participantCount: participants.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── contacts ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function contactDisplayName(card: ContactCard): string {
|
||||
const full = normaliseText(card.name?.full);
|
||||
if (full) return full;
|
||||
const components = card.name?.components ?? [];
|
||||
const ordered = ['prefix', 'given', 'given2', 'additional', 'middle', 'surname', 'surname2', 'suffix'];
|
||||
const byKind = components
|
||||
.slice()
|
||||
.sort((a, b) => ordered.indexOf(a.kind) - ordered.indexOf(b.kind))
|
||||
.map((c) => c.value)
|
||||
.filter((v) => typeof v === 'string' && v.trim().length > 0)
|
||||
.join(' ');
|
||||
if (byKind.trim()) return normaliseText(byKind);
|
||||
const firstEmail = mapValues(card.emails)[0]?.address;
|
||||
if (firstEmail) return firstEmail;
|
||||
const org = mapValues(card.organizations)[0]?.name;
|
||||
return normaliseText(org) || '(unnamed contact)';
|
||||
}
|
||||
|
||||
export function extractContact(jmapAccountId: string, card: ContactCard): IndexDoc {
|
||||
const emails = mapValues(card.emails).map((e) => e.address).filter(Boolean);
|
||||
const phones = mapValues(card.phones).map((p) => p.number).filter(Boolean);
|
||||
const nicknames = mapValues(card.nicknames)
|
||||
.map((n) => n?.name)
|
||||
.filter((v): v is string => typeof v === 'string' && v.length > 0);
|
||||
const orgs = mapValues(card.organizations).map((o) => o.name).filter((v): v is string => !!v);
|
||||
const titles = mapValues(card.titles).map((t) => t.name).filter(Boolean);
|
||||
const notes = mapValues(card.notes).map((n) => n.note).filter(Boolean);
|
||||
// `full` (RFC 9553) when present, else the legacy flat fields vCard import
|
||||
// produces, else the ordered components. All three shapes occur in this type.
|
||||
const addresses = mapValues(card.addresses)
|
||||
.map((a) =>
|
||||
normaliseText(
|
||||
a?.full ||
|
||||
[a?.street, a?.locality, a?.region, a?.postcode, a?.country]
|
||||
.filter((p): p is string => typeof p === 'string' && p.length > 0)
|
||||
.join(', ') ||
|
||||
(a?.components ?? []).map((c) => c.value).join(' '),
|
||||
),
|
||||
)
|
||||
.filter((s) => s.length > 0);
|
||||
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'contact',
|
||||
id: card.id,
|
||||
title: contactDisplayName(card),
|
||||
// Emails/phones go in `people` (weighted above body) because "who is
|
||||
// this / what's their number" is the dominant contact lookup.
|
||||
people: joinUnique([...emails, ...phones, ...nicknames]),
|
||||
body: clamp([...orgs, ...titles, ...addresses, ...notes].filter(Boolean).join('\n')),
|
||||
// A contact has no meaningful single date; JSContact `updated` is optional
|
||||
// and not on this repo's type, so leave it null and rank by relevance only.
|
||||
occurredAt: null,
|
||||
metadata: {
|
||||
kind: card.kind ?? null,
|
||||
emails,
|
||||
phones,
|
||||
organizations: orgs,
|
||||
addressBookIds: Object.keys(card.addressBookIds ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── files ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* METADATA ONLY - filename, path, dates, size, owner. Deliberately NOT file
|
||||
* content: extracting searchable text from arbitrary PDFs / office documents /
|
||||
* images is a materially bigger problem (per-format parsers, OCR, size limits,
|
||||
* untrusted-input parsing in a process holding the user's mail) and is a
|
||||
* separate piece of work. `path` is passed in by the caller because a FileNode
|
||||
* only knows its `parentId`; resolving the chain is the caller's job.
|
||||
*/
|
||||
export function extractFile(
|
||||
jmapAccountId: string,
|
||||
node: FileNode,
|
||||
opts: { path?: string; ownerName?: string } = {},
|
||||
): IndexDoc {
|
||||
const dirPath = normaliseText(opts.path);
|
||||
const isDirectory = node.type === 'd';
|
||||
return {
|
||||
jmapAccountId,
|
||||
contentType: 'file',
|
||||
id: node.id,
|
||||
title: normaliseText(node.name) || '(unnamed file)',
|
||||
people: joinUnique([opts.ownerName, node.accountName]),
|
||||
// The path is genuinely searchable text ("that thing in Invoices/2026"),
|
||||
// and the extension is worth tokenising on its own.
|
||||
body: clamp(
|
||||
[dirPath, isDirectory ? 'folder' : node.type, fileExtension(node.name)]
|
||||
.filter((s) => s && s.length > 0)
|
||||
.join('\n'),
|
||||
),
|
||||
// FileNode has `modified`, NOT `updated` - asking for the wrong name
|
||||
// silently yields undefined (this repo hit that as #700).
|
||||
occurredAt: node.modified ?? node.created ?? null,
|
||||
metadata: {
|
||||
path: dirPath || null,
|
||||
mimeType: isDirectory ? null : node.type,
|
||||
isDirectory,
|
||||
size: typeof node.size === 'number' ? node.size : null,
|
||||
created: node.created ?? null,
|
||||
modified: node.modified ?? null,
|
||||
parentId: node.parentId ?? null,
|
||||
contentIndexed: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fileExtension(name: string | undefined): string {
|
||||
if (!name) return '';
|
||||
const i = name.lastIndexOf('.');
|
||||
return i > 0 && i < name.length - 1 ? name.slice(i + 1).toLowerCase() : '';
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
// A deliberately tiny server-side JMAP client, used only by the indexer.
|
||||
//
|
||||
// WHY NOT REUSE lib/jmap/client.ts: that class is a 7400-line renderer object.
|
||||
// It holds credentials in instance fields, uses `btoa`, opens EventSource /
|
||||
// WebSocket push connections, and wires itself into Zustand stores and toast
|
||||
// notifications. Importing it into an API route would drag all of that into the
|
||||
// server bundle for the sake of four method calls. The existing server-side
|
||||
// JMAP code in this repo (lib/auth/verify-jmap-auth.ts) already sets the
|
||||
// precedent: plain fetch + an Authorization header.
|
||||
//
|
||||
// Everything here is stateless - the caller supplies the auth header per call,
|
||||
// so there is no resident credential and nothing to invalidate.
|
||||
|
||||
import type { CalendarEvent, ContactCard, Email, FileNode } from '@/lib/jmap/types';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 30_000;
|
||||
|
||||
export const CAP_CORE = 'urn:ietf:params:jmap:core';
|
||||
export const CAP_MAIL = 'urn:ietf:params:jmap:mail';
|
||||
export const CAP_CALENDARS = 'urn:ietf:params:jmap:calendars';
|
||||
export const CAP_CONTACTS = 'urn:ietf:params:jmap:contacts';
|
||||
export const CAP_FILENODE = 'urn:ietf:params:jmap:filenode';
|
||||
|
||||
export class JmapIndexError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.name = 'JmapIndexError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export interface JmapSessionInfo {
|
||||
apiUrl: string;
|
||||
/** Server-confirmed authenticated login (JMAP Session.username). */
|
||||
username?: string;
|
||||
primaryAccounts: Record<string, string>;
|
||||
accounts: Record<string, { name?: string; isPersonal?: boolean; accountCapabilities?: Record<string, unknown> }>;
|
||||
capabilities: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pins a URL advertised by the session to the origin we authenticated against.
|
||||
*
|
||||
* `lib/jmap/client.ts` does the same thing in its rewriteSessionUrls() for the
|
||||
* renderer's benefit. Server-side it is a security control, not a convenience:
|
||||
* we attach the user's credentials to this URL, so a session document that
|
||||
* advertised an `apiUrl` on someone else's host would turn this into a
|
||||
* credential-leaking SSRF. Keep the path and query, take the origin from the
|
||||
* server URL we were configured with.
|
||||
*/
|
||||
function pinToServerOrigin(advertised: string, serverUrl: string): string {
|
||||
const base = new URL(serverUrl);
|
||||
let target: URL;
|
||||
try {
|
||||
target = new URL(advertised, base);
|
||||
} catch {
|
||||
throw new JmapIndexError('JMAP session advertised an unusable apiUrl');
|
||||
}
|
||||
return `${base.origin}${target.pathname}${target.search}`;
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
return await fetch(url, { ...init, signal: controller.signal, redirect: 'manual' });
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
throw new JmapIndexError('JMAP request timed out', 504);
|
||||
}
|
||||
throw new JmapIndexError(`JMAP request failed: ${String(error)}`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchJmapSession(serverUrl: string, authHeader: string): Promise<JmapSessionInfo> {
|
||||
const response = await fetchWithTimeout(`${serverUrl.replace(/\/+$/, '')}/.well-known/jmap`, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: authHeader },
|
||||
});
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new JmapIndexError('JMAP authentication failed', 401);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new JmapIndexError(`JMAP session fetch failed (${response.status})`);
|
||||
}
|
||||
const raw = (await response.json().catch(() => null)) as Record<string, unknown> | null;
|
||||
if (!raw || typeof raw.apiUrl !== 'string') {
|
||||
throw new JmapIndexError('Invalid JMAP session response');
|
||||
}
|
||||
return {
|
||||
apiUrl: pinToServerOrigin(raw.apiUrl, serverUrl),
|
||||
username: typeof raw.username === 'string' ? raw.username : undefined,
|
||||
primaryAccounts: (raw.primaryAccounts as Record<string, string>) ?? {},
|
||||
accounts: (raw.accounts as JmapSessionInfo['accounts']) ?? {},
|
||||
capabilities: (raw.capabilities as Record<string, unknown>) ?? {},
|
||||
};
|
||||
}
|
||||
|
||||
type MethodCall = [string, Record<string, unknown>, string];
|
||||
|
||||
/** Raw method-response tuple, `[name, args, callId]`. `name` is 'error' on failure. */
|
||||
type MethodResponse = [string, Record<string, unknown>, string];
|
||||
|
||||
export async function jmapRequest(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
using: readonly string[],
|
||||
methodCalls: readonly MethodCall[],
|
||||
): Promise<MethodResponse[]> {
|
||||
const response = await fetchWithTimeout(session.apiUrl, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: authHeader, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ using, methodCalls }),
|
||||
});
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new JmapIndexError('JMAP authentication failed', 401);
|
||||
}
|
||||
if (response.status === 429) {
|
||||
throw new JmapIndexError('JMAP server is rate limiting', 429);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new JmapIndexError(`JMAP request failed (${response.status})`);
|
||||
}
|
||||
const data = (await response.json().catch(() => null)) as { methodResponses?: MethodResponse[] } | null;
|
||||
if (!data || !Array.isArray(data.methodResponses)) {
|
||||
throw new JmapIndexError('Invalid JMAP response envelope');
|
||||
}
|
||||
return data.methodResponses;
|
||||
}
|
||||
|
||||
function firstResult(responses: MethodResponse[], expected: string): Record<string, unknown> | null {
|
||||
for (const [name, args] of responses) {
|
||||
if (name === expected) return args;
|
||||
// A method-level error is not fatal for an INDEX: a server that doesn't
|
||||
// support one data type should not fail the whole reindex. The caller
|
||||
// treats null as "nothing to index for this type".
|
||||
if (name === 'error') return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function idsOf(args: Record<string, unknown> | null): string[] {
|
||||
const ids = args?.ids;
|
||||
return Array.isArray(ids) ? ids.filter((v): v is string => typeof v === 'string') : [];
|
||||
}
|
||||
|
||||
function listOf<T>(args: Record<string, unknown> | null): T[] {
|
||||
const list = args?.list;
|
||||
return Array.isArray(list) ? (list as T[]) : [];
|
||||
}
|
||||
|
||||
export function accountIdFor(session: JmapSessionInfo, capability: string): string | null {
|
||||
const id = session.primaryAccounts[capability];
|
||||
return typeof id === 'string' && id.length > 0 ? id : null;
|
||||
}
|
||||
|
||||
export function hasCapability(session: JmapSessionInfo, capability: string): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(session.capabilities, capability);
|
||||
}
|
||||
|
||||
/** Per-ACCOUNT capability, mirroring client.ts's supportsFiles() (#563: a server can advertise it while an account has it revoked). */
|
||||
export function accountHasCapability(
|
||||
session: JmapSessionInfo,
|
||||
accountId: string,
|
||||
capability: string,
|
||||
): boolean {
|
||||
const account = session.accounts[accountId];
|
||||
if (!account) return false;
|
||||
if (account.accountCapabilities && Object.prototype.hasOwnProperty.call(account.accountCapabilities, capability)) {
|
||||
return true;
|
||||
}
|
||||
return account.isPersonal === false;
|
||||
}
|
||||
|
||||
// ── mail ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Properties needed to build a mail IndexDoc. Bodies come via bodyValues. */
|
||||
const EMAIL_INDEX_PROPERTIES = [
|
||||
'id', 'threadId', 'mailboxIds', 'keywords', 'size', 'receivedAt',
|
||||
'from', 'to', 'cc', 'subject', 'preview', 'hasAttachment',
|
||||
'textBody', 'htmlBody', 'bodyValues',
|
||||
] as const;
|
||||
|
||||
export async function getEmailsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
maxBodyBytes: number,
|
||||
): Promise<Email[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||
['Email/get', {
|
||||
accountId,
|
||||
ids: [...ids],
|
||||
properties: [...EMAIL_INDEX_PROPERTIES],
|
||||
// Without these two the bodyValues map comes back EMPTY and every
|
||||
// indexed body would silently fall back to `preview`.
|
||||
fetchTextBodyValues: true,
|
||||
fetchHTMLBodyValues: true,
|
||||
maxBodyValueBytes: maxBodyBytes,
|
||||
}, 'g'],
|
||||
]);
|
||||
return listOf<Email>(firstResult(responses, 'Email/get'));
|
||||
}
|
||||
|
||||
export async function queryRecentEmailIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
afterIso: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||
['Email/query', {
|
||||
accountId,
|
||||
filter: { after: afterIso },
|
||||
sort: [{ property: 'receivedAt', isAscending: false }],
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
}, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'Email/query'));
|
||||
}
|
||||
|
||||
// ── calendar ────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function getCalendarEventsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<CalendarEvent[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
||||
['CalendarEvent/get', { accountId, ids: [...ids] }, 'g'],
|
||||
]);
|
||||
return listOf<CalendarEvent>(firstResult(responses, 'CalendarEvent/get'));
|
||||
}
|
||||
|
||||
export async function queryCalendarEventIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
afterIso: string,
|
||||
beforeIso: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
||||
['CalendarEvent/query', {
|
||||
accountId,
|
||||
// LocalDateTime, per the note in lib/jmap/client.ts:307-312 - Stalwart
|
||||
// parses these without a zone suffix and ignores unparseable values.
|
||||
filter: { after: toLocalDateTime(afterIso), before: toLocalDateTime(beforeIso) },
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
}, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'CalendarEvent/query'));
|
||||
}
|
||||
|
||||
/** JSCalendar LocalDateTime: `YYYY-MM-DDTHH:MM:SS`, no zone designator. */
|
||||
function toLocalDateTime(iso: string): string {
|
||||
return iso.replace(/\.\d+/, '').replace(/Z$/, '').slice(0, 19);
|
||||
}
|
||||
|
||||
// ── contacts ────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function getContactsForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<ContactCard[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
||||
['ContactCard/get', { accountId, ids: [...ids] }, 'g'],
|
||||
]);
|
||||
return listOf<ContactCard>(firstResult(responses, 'ContactCard/get'));
|
||||
}
|
||||
|
||||
export async function queryContactIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
||||
['ContactCard/query', { accountId, limit, calculateTotal: false }, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'ContactCard/query'));
|
||||
}
|
||||
|
||||
// ── files ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const FILENODE_INDEX_PROPERTIES = [
|
||||
'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified',
|
||||
] as const;
|
||||
|
||||
export async function getFilesForIndex(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
ids: readonly string[],
|
||||
): Promise<FileNode[]> {
|
||||
if (ids.length === 0) return [];
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
||||
['FileNode/get', { accountId, ids: [...ids], properties: [...FILENODE_INDEX_PROPERTIES] }, 'g'],
|
||||
]);
|
||||
return listOf<FileNode>(firstResult(responses, 'FileNode/get'));
|
||||
}
|
||||
|
||||
export async function queryFileIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
||||
['FileNode/query', { accountId, filter: {}, limit, calculateTotal: false }, 'q'],
|
||||
]);
|
||||
return idsOf(firstResult(responses, 'FileNode/query'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds `id -> "Parent/Child"` paths for the given nodes, walking `parentId`
|
||||
* upward. FileNode only knows its parent, so the caller has to assemble this;
|
||||
* unresolvable ancestors just truncate the path rather than failing.
|
||||
*/
|
||||
export function buildFilePaths(nodes: readonly FileNode[]): Map<string, string> {
|
||||
const byId = new Map(nodes.map((n) => [n.id, n]));
|
||||
const cache = new Map<string, string>();
|
||||
|
||||
const resolve = (id: string, depth: number): string => {
|
||||
if (depth > 32) return '';
|
||||
const cached = cache.get(id);
|
||||
if (cached !== undefined) return cached;
|
||||
const node = byId.get(id);
|
||||
if (!node) return '';
|
||||
const parent = node.parentId ? resolve(node.parentId, depth + 1) : '';
|
||||
const full = parent ? `${parent}/${node.name}` : node.name;
|
||||
cache.set(id, full);
|
||||
return full;
|
||||
};
|
||||
|
||||
const out = new Map<string, string>();
|
||||
for (const n of nodes) {
|
||||
// The document's own `path` metadata is its PARENT directory chain, so a
|
||||
// search for "Invoices" matches files inside it without the filename
|
||||
// being duplicated into the body.
|
||||
out.set(n.id, n.parentId ? resolve(n.parentId, 0) : '');
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Server-side client for the main process's key service (electron/key-service.ts).
|
||||
//
|
||||
// Asks for an account's index key over the inherited fd only when a job needs
|
||||
// it, and drops it as soon as the job finishes. There is deliberately no cache:
|
||||
// a resident plaintext key in a long-lived process is exactly the thing the OS
|
||||
// keychain exists to avoid, and a keychain round trip costs microseconds
|
||||
// against a job that makes network calls.
|
||||
|
||||
import net from 'node:net';
|
||||
|
||||
/** Set by electron/main.ts alongside VNCMAIL_DESKTOP_STORE_DIR. */
|
||||
export const KEY_FD_ENV = 'VNCMAIL_DESKTOP_KEY_FD';
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
export type KeyErrorCode =
|
||||
| 'no-channel'
|
||||
| 'no-secure-storage'
|
||||
| 'key-io-failed'
|
||||
| 'key-unreadable'
|
||||
| 'bad-request'
|
||||
| 'timeout';
|
||||
|
||||
export class IndexKeyError extends Error {
|
||||
code: KeyErrorCode;
|
||||
constructor(code: KeyErrorCode, message: string) {
|
||||
super(message);
|
||||
this.name = 'IndexKeyError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
interface Pending {
|
||||
resolve: (value: { key?: string }) => void;
|
||||
reject: (error: Error) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
let socket: net.Socket | null = null;
|
||||
let nextId = 1;
|
||||
const pending = new Map<number, Pending>();
|
||||
let readBuffer = '';
|
||||
|
||||
function failAll(error: Error): void {
|
||||
for (const [, p] of pending) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(error);
|
||||
}
|
||||
pending.clear();
|
||||
}
|
||||
|
||||
function getSocket(): net.Socket {
|
||||
if (socket && !socket.destroyed) return socket;
|
||||
|
||||
const raw = process.env[KEY_FD_ENV]?.trim();
|
||||
const fd = raw ? Number(raw) : NaN;
|
||||
if (!Number.isInteger(fd) || fd < 3) {
|
||||
throw new IndexKeyError(
|
||||
'no-channel',
|
||||
`${KEY_FD_ENV} is not a usable file descriptor (got ${JSON.stringify(raw)}). ` +
|
||||
`The local index only works inside the Electron desktop shell.`,
|
||||
);
|
||||
}
|
||||
|
||||
let created: net.Socket;
|
||||
try {
|
||||
created = new net.Socket({ fd, readable: true, writable: true });
|
||||
} catch (error) {
|
||||
throw new IndexKeyError('no-channel', `Could not open fd ${fd}: ${String(error)}`);
|
||||
}
|
||||
// The channel outlives every individual request; don't let it hold the event
|
||||
// loop open on its own.
|
||||
created.unref();
|
||||
|
||||
created.on('data', (chunk: Buffer) => {
|
||||
readBuffer += chunk.toString('utf8');
|
||||
if (readBuffer.length > 64 * 1024) readBuffer = '';
|
||||
let newline: number;
|
||||
while ((newline = readBuffer.indexOf('\n')) >= 0) {
|
||||
const line = readBuffer.slice(0, newline);
|
||||
readBuffer = readBuffer.slice(newline + 1);
|
||||
if (!line.trim()) continue;
|
||||
let msg: { id?: unknown; ok?: unknown; key?: unknown; code?: unknown; error?: unknown };
|
||||
try {
|
||||
msg = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const id = typeof msg.id === 'number' ? msg.id : null;
|
||||
if (id === null) continue;
|
||||
const p = pending.get(id);
|
||||
if (!p) continue;
|
||||
pending.delete(id);
|
||||
clearTimeout(p.timer);
|
||||
if (msg.ok === true) {
|
||||
p.resolve({ key: typeof msg.key === 'string' ? msg.key : undefined });
|
||||
} else {
|
||||
const code = typeof msg.code === 'string' ? (msg.code as KeyErrorCode) : 'key-io-failed';
|
||||
p.reject(new IndexKeyError(code, typeof msg.error === 'string' ? msg.error : 'Key request failed'));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const onGone = (error?: Error) => {
|
||||
socket = null;
|
||||
readBuffer = '';
|
||||
failAll(error ?? new IndexKeyError('no-channel', 'Key service channel closed'));
|
||||
};
|
||||
created.on('close', () => onGone());
|
||||
created.on('error', (error) => onGone(new IndexKeyError('no-channel', String(error))));
|
||||
|
||||
socket = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
function request(op: 'getIndexKey' | 'deleteIndexKey', accountId: string): Promise<{ key?: string }> {
|
||||
const sock = getSocket();
|
||||
const id = nextId++;
|
||||
return new Promise<{ key?: string }>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
pending.delete(id);
|
||||
reject(new IndexKeyError('timeout', `Key service did not answer within ${REQUEST_TIMEOUT_MS}ms`));
|
||||
}, REQUEST_TIMEOUT_MS);
|
||||
// Don't let a pending key request keep the process alive either.
|
||||
timer.unref?.();
|
||||
pending.set(id, { resolve, reject, timer });
|
||||
try {
|
||||
sock.write(`${JSON.stringify({ id, op, accountId })}\n`);
|
||||
} catch (error) {
|
||||
pending.delete(id);
|
||||
clearTimeout(timer);
|
||||
reject(new IndexKeyError('no-channel', `Could not write to the key service: ${String(error)}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `fn` with the account's raw index key, then zeroes the buffer.
|
||||
*
|
||||
* Zeroing a Buffer is genuine (unlike a JS string, which cannot be scrubbed) -
|
||||
* which is why the key crosses the boundary as hex and is converted to a Buffer
|
||||
* exactly once, here. `store.ts` puts the hex into a `PRAGMA` string, so a copy
|
||||
* does briefly exist in the JS heap; the buffer wipe bounds how long the
|
||||
* long-lived copy lives, it does not pretend to eliminate every trace.
|
||||
*/
|
||||
export async function withIndexKey<T>(
|
||||
accountId: string,
|
||||
fn: (key: Buffer) => Promise<T> | T,
|
||||
): Promise<T> {
|
||||
const { key: hex } = await request('getIndexKey', accountId);
|
||||
if (!hex) throw new IndexKeyError('key-io-failed', 'Key service returned no key');
|
||||
const key = Buffer.from(hex, 'hex');
|
||||
if (key.length !== 32) {
|
||||
key.fill(0);
|
||||
throw new IndexKeyError('key-io-failed', `Key service returned ${key.length} bytes, expected 32`);
|
||||
}
|
||||
try {
|
||||
return await fn(key);
|
||||
} finally {
|
||||
key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
/** Used when purging an account: the key goes FIRST, so an interrupted purge leaves unreadable data. */
|
||||
export async function deleteIndexKey(accountId: string): Promise<void> {
|
||||
await request('deleteIndexKey', accountId);
|
||||
}
|
||||
|
||||
/** True when this process has a key channel at all (i.e. is the desktop shell's server). */
|
||||
export function hasKeyChannel(): boolean {
|
||||
const raw = process.env[KEY_FD_ENV]?.trim();
|
||||
const fd = raw ? Number(raw) : NaN;
|
||||
return Number.isInteger(fd) && fd >= 3;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// The hosted-deployment gate, and where an account's index file lives.
|
||||
//
|
||||
// The standalone Next.js server in `electron/main.ts` is the SAME artifact the
|
||||
// production `Dockerfile` ships to multi-tenant deployments. An index that
|
||||
// activated unconditionally would have a shared server start writing every
|
||||
// user's mail into a server-side SQLite file. So activation is keyed on an env
|
||||
// var that ONLY `electron/main.ts` sets, and that same var supplies the path -
|
||||
// one variable doing both jobs, so they cannot drift apart.
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
|
||||
/** Set by electron/main.ts on spawn. Absent => the feature does not exist. */
|
||||
export const STORE_DIR_ENV = 'VNCMAIL_DESKTOP_STORE_DIR';
|
||||
|
||||
/**
|
||||
* The index root, or `null` when this process is not the desktop shell's
|
||||
* server. Every route must 404 on `null` - not 403, since nothing should learn
|
||||
* the routes exist in a deployment that doesn't have the feature.
|
||||
*/
|
||||
export function getStoreDir(): string | null {
|
||||
const dir = process.env[STORE_DIR_ENV]?.trim();
|
||||
if (!dir) return null;
|
||||
// Must be absolute: a relative path would resolve against the server's cwd,
|
||||
// which differs between `electron:dev` and a packaged build.
|
||||
if (!path.isAbsolute(dir)) return null;
|
||||
return dir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filenames are a hash, not `username@host`, so a directory listing is not a
|
||||
* plaintext inventory of the user's accounts. The account id itself lives only
|
||||
* inside the encrypted file (and in the renderer's own `account-registry`,
|
||||
* which already stores it in plain localStorage).
|
||||
*/
|
||||
export function accountFileToken(accountId: string): string {
|
||||
return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
export function indexDbPath(storeDir: string, accountId: string): string {
|
||||
return path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`);
|
||||
}
|
||||
|
||||
export function keyFilePath(storeDir: string, accountId: string): string {
|
||||
return path.join(storeDir, 'keys', `${accountFileToken(accountId)}.bin`);
|
||||
}
|
||||
|
||||
/** WAL siblings must be removed with the database, or a purge leaks readable pages. */
|
||||
export function dbSiblings(dbPath: string): string[] {
|
||||
return [dbPath, `${dbPath}-wal`, `${dbPath}-shm`];
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
// The index jobs.
|
||||
//
|
||||
// TWO SHAPES, both plain request-scoped work - there is no background worker,
|
||||
// no cursor, no retry ladder and no resident credential anywhere:
|
||||
//
|
||||
// 1. `indexDocuments()` - the PRIMARY path. The renderer's live JMAP push
|
||||
// connection sees a StateChange, and calls the route with the ids that
|
||||
// changed (or with no ids, meaning "refetch what's recent for this type").
|
||||
// One or a handful of objects, fetched and upserted.
|
||||
// 2. `catchUpAll()` - the FALLBACK. On app launch, backfill a bounded recent
|
||||
// window for every supported type, because anything that changed while the
|
||||
// app was closed produced no push event.
|
||||
//
|
||||
// Staleness between refreshes is acceptable by design: this is a search index
|
||||
// for a retrieval/AI feature, not a mail replica.
|
||||
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { generateAccountId } from '@/lib/account-utils';
|
||||
import { getStalwartCredentials } from '@/lib/stalwart/credentials';
|
||||
import { logger } from '@/lib/logger';
|
||||
import {
|
||||
accountHasCapability, accountIdFor, buildFilePaths, CAP_CALENDARS, CAP_CONTACTS,
|
||||
CAP_FILENODE, CAP_MAIL, fetchJmapSession, getCalendarEventsForIndex, getContactsForIndex,
|
||||
getEmailsForIndex, getFilesForIndex, hasCapability, JmapIndexError, queryCalendarEventIds,
|
||||
queryContactIds, queryFileIds, queryRecentEmailIds, type JmapSessionInfo,
|
||||
} from './jmap';
|
||||
import { extractCalendarEvent, extractContact, extractFile, extractMail } from './extract';
|
||||
import { withIndexKey } from './key';
|
||||
import { getStoreDir } from './paths';
|
||||
import { MailIndex, type ContentType, type IndexDoc } from './store';
|
||||
|
||||
/**
|
||||
* Bounded window. Small on purpose: this is the first cut of a retrieval index,
|
||||
* and a wide window turns "index on every delivery" into a slow request. The
|
||||
* event-driven path indexes single objects, so the window only bounds catch-up.
|
||||
*/
|
||||
export const INDEX_WINDOW_DAYS = 30;
|
||||
/** Calendar looks forward as well as back - upcoming events are the useful ones. */
|
||||
export const CALENDAR_FORWARD_DAYS = 180;
|
||||
/** Per-type ceiling for one catch-up pass. */
|
||||
export const CATCHUP_MAX_PER_TYPE = 500;
|
||||
/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
|
||||
export const MAX_IDS_PER_CALL = 200;
|
||||
/** Cap on body bytes requested per message from the server. */
|
||||
export const MAX_BODY_VALUE_BYTES = 256_000;
|
||||
/** Contacts and files have no useful date filter, so they are simply capped. */
|
||||
export const CONTACTS_MAX = 2_000;
|
||||
export const FILES_MAX = 2_000;
|
||||
|
||||
export interface IndexSession {
|
||||
serverUrl: string;
|
||||
authHeader: string;
|
||||
username: string;
|
||||
slot: number;
|
||||
/** `username@host` - the durable per-account key. NEVER the cookie slot. */
|
||||
accountId: string;
|
||||
}
|
||||
|
||||
export class IndexSessionError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status: number) {
|
||||
super(message);
|
||||
this.name = 'IndexSessionError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the calling request to an account and a usable Authorization header.
|
||||
*
|
||||
* Uses the SAME per-slot encrypted `jmap_stalwart_ctx` cookie that
|
||||
* `/api/settings`, `/api/push/preview` and `/api/plugin-approval-status`
|
||||
* already read (`lib/stalwart/credentials.ts`). That cookie is written by
|
||||
* `/api/auth/stalwart-context`, which the renderer syncs on every login,
|
||||
* session restore, SSO callback, account switch and token refresh
|
||||
* (`stores/auth-store.ts`, 10 call sites), and it carries a ready-made header
|
||||
* for BOTH basic and bearer accounts.
|
||||
*
|
||||
* Why this matters beyond convenience: it means the indexer never touches the
|
||||
* OAuth refresh-token cookie. A server-side refresh would rotate the token into
|
||||
* a response nobody reads while the browser kept the superseded one, and the
|
||||
* next real refresh would then fail and log the user out. Reading an
|
||||
* already-minted header cannot cause that.
|
||||
*/
|
||||
export async function resolveIndexSession(request: NextRequest): Promise<IndexSession> {
|
||||
const credentials = await getStalwartCredentials(request);
|
||||
if (!credentials) {
|
||||
throw new IndexSessionError('No JMAP auth context for this account; sign in again.', 401);
|
||||
}
|
||||
const accountId = generateAccountId(credentials.username, credentials.serverUrl);
|
||||
return { ...credentials, accountId };
|
||||
}
|
||||
|
||||
export interface IndexResult {
|
||||
accountId: string;
|
||||
/** Per-type counts of documents written. */
|
||||
written: Partial<Record<ContentType, number>>;
|
||||
/** Types the server (or this account) doesn't support, so nothing was attempted. */
|
||||
skipped: ContentType[];
|
||||
/** Non-fatal per-type failures. One broken type must not fail the whole call. */
|
||||
errors: Array<{ contentType: ContentType; message: string }>;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
function isoDaysFromNow(days: number): string {
|
||||
return new Date(Date.now() + days * 24 * 60 * 60 * 1000).toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Which types this session can actually index. Calendar/contacts are session
|
||||
* capabilities; files is a PER-ACCOUNT capability (a server can advertise
|
||||
* filenode while a specific account has it revoked - #563).
|
||||
*/
|
||||
export function supportedTypes(session: JmapSessionInfo): {
|
||||
supported: ContentType[];
|
||||
skipped: ContentType[];
|
||||
accountIds: Partial<Record<ContentType, string>>;
|
||||
} {
|
||||
const supported: ContentType[] = [];
|
||||
const skipped: ContentType[] = [];
|
||||
const accountIds: Partial<Record<ContentType, string>> = {};
|
||||
|
||||
const mailAccount = accountIdFor(session, CAP_MAIL);
|
||||
if (mailAccount) { supported.push('mail'); accountIds.mail = mailAccount; }
|
||||
else skipped.push('mail');
|
||||
|
||||
const calAccount = accountIdFor(session, CAP_CALENDARS);
|
||||
if (calAccount && hasCapability(session, CAP_CALENDARS)) {
|
||||
supported.push('calendar'); accountIds.calendar = calAccount;
|
||||
} else skipped.push('calendar');
|
||||
|
||||
const contactAccount = accountIdFor(session, CAP_CONTACTS);
|
||||
if (contactAccount && hasCapability(session, CAP_CONTACTS)) {
|
||||
supported.push('contact'); accountIds.contact = contactAccount;
|
||||
} else skipped.push('contact');
|
||||
|
||||
// Files fall back to the mail account id: Stalwart exposes FileNode on the
|
||||
// same account and does not always list a primaryAccounts entry for it.
|
||||
const fileAccount = accountIdFor(session, CAP_FILENODE) ?? mailAccount;
|
||||
if (fileAccount && accountHasCapability(session, fileAccount, CAP_FILENODE)) {
|
||||
supported.push('file'); accountIds.file = fileAccount;
|
||||
} else skipped.push('file');
|
||||
|
||||
return { supported, skipped, accountIds };
|
||||
}
|
||||
|
||||
interface FetchArgs {
|
||||
session: JmapSessionInfo;
|
||||
authHeader: string;
|
||||
jmapAccountId: string;
|
||||
ids: readonly string[] | null;
|
||||
}
|
||||
|
||||
/** Fetches and flattens one content type. `ids === null` means "the recent window". */
|
||||
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<IndexDoc[]> {
|
||||
const { session, authHeader, jmapAccountId, ids } = args;
|
||||
|
||||
switch (contentType) {
|
||||
case 'mail': {
|
||||
const targetIds = ids ?? await queryRecentEmailIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
// Chunked because bodies are big: one Email/get for 500 messages with
|
||||
// full bodies would be an enormous response.
|
||||
for (let i = 0; i < targetIds.length; i += 25) {
|
||||
const emails = await getEmailsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 25), MAX_BODY_VALUE_BYTES,
|
||||
);
|
||||
for (const email of emails) docs.push(extractMail(jmapAccountId, email));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'calendar': {
|
||||
const targetIds = ids ?? await queryCalendarEventIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||
CATCHUP_MAX_PER_TYPE,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 50) {
|
||||
const events = await getCalendarEventsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 50),
|
||||
);
|
||||
for (const event of events) docs.push(extractCalendarEvent(jmapAccountId, event));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'contact': {
|
||||
const targetIds = ids ?? await queryContactIds(session, authHeader, jmapAccountId, CONTACTS_MAX);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 100) {
|
||||
const cards = await getContactsForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
|
||||
);
|
||||
for (const card of cards) docs.push(extractContact(jmapAccountId, card));
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
case 'file': {
|
||||
const targetIds = ids ?? await queryFileIds(session, authHeader, jmapAccountId, FILES_MAX);
|
||||
const nodes = [];
|
||||
for (let i = 0; i < targetIds.length; i += 100) {
|
||||
nodes.push(...await getFilesForIndex(
|
||||
session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
|
||||
));
|
||||
}
|
||||
// Paths need the whole set in hand, so this one can't stream per chunk.
|
||||
const paths = buildFilePaths(nodes);
|
||||
return nodes
|
||||
// Directories are indexed too: "what's in the Invoices folder" is a
|
||||
// real query, and a folder row is a few bytes.
|
||||
.map((node) => extractFile(jmapAccountId, node, { path: paths.get(node.id) }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface IndexRequest {
|
||||
/** Types to touch. Empty means every supported type. */
|
||||
types?: readonly ContentType[];
|
||||
/**
|
||||
* Per-type ids to index. Omitted/empty for a type means "refetch that type's
|
||||
* recent window" (the catch-up shape).
|
||||
*/
|
||||
ids?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Per-type ids to REMOVE (a JMAP `destroyed`). */
|
||||
removed?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Drop documents outside the retention window after writing. */
|
||||
prune?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one index pass. Opens the encrypted store, fetches, upserts, closes.
|
||||
*
|
||||
* The key is fetched from the main process for the duration of this call only
|
||||
* (`withIndexKey`) and zeroed afterwards - there is no cached handle and no
|
||||
* resident key.
|
||||
*/
|
||||
export async function runIndex(
|
||||
indexSession: IndexSession,
|
||||
req: IndexRequest,
|
||||
): Promise<IndexResult> {
|
||||
const started = Date.now();
|
||||
const storeDir = getStoreDir();
|
||||
if (!storeDir) {
|
||||
throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
|
||||
}
|
||||
|
||||
const session = await fetchJmapSession(indexSession.serverUrl, indexSession.authHeader);
|
||||
|
||||
// Identity cross-check. `generateAccountId` used the username from the auth
|
||||
// context cookie; the server may canonicalise a short login (`linus`) to a
|
||||
// full address (`linus@example.com`) - which is exactly why AccountEntry
|
||||
// carries `serverIdentifiers`. Accept either form, reject anything else
|
||||
// rather than writing one account's mail into another's file.
|
||||
if (session.username) {
|
||||
const serverAccountId = generateAccountId(session.username, indexSession.serverUrl);
|
||||
if (serverAccountId !== indexSession.accountId) {
|
||||
const shortMatches = session.username.split('@')[0] === indexSession.username.split('@')[0];
|
||||
if (!shortMatches) {
|
||||
throw new IndexSessionError(
|
||||
'The JMAP session belongs to a different account than the request cookie.',
|
||||
409,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { supported, skipped, accountIds } = supportedTypes(session);
|
||||
const requested = req.types && req.types.length > 0 ? req.types : supported;
|
||||
const types = requested.filter((t) => supported.includes(t));
|
||||
const notAttempted = [...new Set([...skipped, ...requested.filter((t) => !supported.includes(t))])];
|
||||
|
||||
const written: Partial<Record<ContentType, number>> = {};
|
||||
const errors: IndexResult['errors'] = [];
|
||||
|
||||
await withIndexKey(indexSession.accountId, async (key) => {
|
||||
const index = MailIndex.open({ storeDir, accountId: indexSession.accountId, key });
|
||||
try {
|
||||
for (const contentType of types) {
|
||||
const jmapAccountId = accountIds[contentType];
|
||||
if (!jmapAccountId) continue;
|
||||
try {
|
||||
const removed = req.removed?.[contentType];
|
||||
if (removed && removed.length > 0) {
|
||||
index.remove(jmapAccountId, contentType, removed.slice(0, MAX_IDS_PER_CALL));
|
||||
}
|
||||
|
||||
const requestedIds = req.ids?.[contentType];
|
||||
const ids = requestedIds && requestedIds.length > 0
|
||||
? requestedIds.slice(0, MAX_IDS_PER_CALL)
|
||||
: null;
|
||||
|
||||
const docs = await fetchDocs(contentType, {
|
||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids,
|
||||
});
|
||||
written[contentType] = index.upsert(docs);
|
||||
|
||||
if (req.prune && contentType === 'mail') {
|
||||
// Only mail prunes by date: calendar's window looks forward,
|
||||
// contacts have no date, and file rows are metadata-sized.
|
||||
index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS));
|
||||
}
|
||||
} catch (error) {
|
||||
// One unsupported or misbehaving type must not fail the others.
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push({ contentType, message });
|
||||
if (error instanceof JmapIndexError && error.status === 401) throw error;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
index.close();
|
||||
}
|
||||
});
|
||||
|
||||
const result: IndexResult = {
|
||||
accountId: indexSession.accountId,
|
||||
written,
|
||||
skipped: notAttempted,
|
||||
errors,
|
||||
durationMs: Date.now() - started,
|
||||
};
|
||||
logger.info('mail-index: pass complete', {
|
||||
slot: indexSession.slot,
|
||||
written: JSON.stringify(written),
|
||||
skipped: notAttempted.join(',') || 'none',
|
||||
errors: errors.length,
|
||||
durationMs: result.durationMs,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
// The encrypted local search index: schema, open/close, upsert, search.
|
||||
//
|
||||
// One SQLite (SQLCipher) file per account. Rows are ALSO account-scoped
|
||||
// internally - `(jmap_account_id, content_type, id)` - because a single login
|
||||
// exposes the user's own JMAP account plus every delegated/shared account, and
|
||||
// JMAP ids are unique only WITHIN an account (this codebase already works
|
||||
// around that collision in `lib/jmap/client.ts:388`'s namespaceMailboxIds).
|
||||
// One file per account keeps purge trivial; the composite key keeps
|
||||
// delegated accounts from merging inside it.
|
||||
//
|
||||
// This is a SEARCH INDEX, not a mail replica. It is allowed to be stale, it is
|
||||
// allowed to be incomplete, and it can be discarded and rebuilt at any time -
|
||||
// which is why the schema-version mismatch path below simply drops everything
|
||||
// rather than migrating.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { loadSqlcipher, type SqlcipherDatabase } from './binding';
|
||||
import { dbSiblings, indexDbPath } from './paths';
|
||||
|
||||
export const SCHEMA_VERSION = 1;
|
||||
|
||||
export type ContentType = 'mail' | 'calendar' | 'contact' | 'file';
|
||||
|
||||
export const CONTENT_TYPES: readonly ContentType[] = ['mail', 'calendar', 'contact', 'file'];
|
||||
|
||||
export function isContentType(v: unknown): v is ContentType {
|
||||
return typeof v === 'string' && (CONTENT_TYPES as readonly string[]).includes(v);
|
||||
}
|
||||
|
||||
/**
|
||||
* One indexable thing, already flattened to text. Produced by the pure
|
||||
* extractors in `extract.ts` so that every JMAP-shape decision is unit-testable
|
||||
* without a database or a server.
|
||||
*/
|
||||
export interface IndexDoc {
|
||||
jmapAccountId: string;
|
||||
contentType: ContentType;
|
||||
/** JMAP id. Unique only within (jmapAccountId, contentType). */
|
||||
id: string;
|
||||
/** Subject / event title / contact display name / filename. */
|
||||
title: string;
|
||||
/** Addresses and names: sender+recipients, attendees, contact emails/phones, owner. */
|
||||
people: string;
|
||||
/** The bulk searchable text. Plain text only - never HTML. */
|
||||
body: string;
|
||||
/** ISO 8601, or null when the type has no meaningful date. Drives recency ordering. */
|
||||
occurredAt: string | null;
|
||||
/** Small type-specific extras returned verbatim to the caller (never searched). */
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface SearchHit {
|
||||
contentType: ContentType;
|
||||
id: string;
|
||||
jmapAccountId: string;
|
||||
title: string;
|
||||
people: string;
|
||||
occurredAt: string | null;
|
||||
metadata: Record<string, unknown>;
|
||||
/** FTS5 bm25 score. Lower is a better match (bm25 returns negative values). */
|
||||
score: number;
|
||||
/** Highlighted excerpt from the body, for feeding an LLM as context. */
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
const DDL = `
|
||||
CREATE TABLE IF NOT EXISTS doc (
|
||||
jmap_account_id TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
people TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
occurred_at TEXT,
|
||||
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||
indexed_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (jmap_account_id, content_type, id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS doc_recent
|
||||
ON doc(jmap_account_id, content_type, occurred_at DESC);
|
||||
|
||||
-- Standalone (not external-content) FTS5: the text is duplicated into this
|
||||
-- table and kept in step manually on upsert. External content would avoid the
|
||||
-- duplication but requires deleting the old FTS row using its OLD column
|
||||
-- values, which an upsert does not have to hand - a well-known source of
|
||||
-- silently-stale FTS rows. At this scale (a bounded recent window) the
|
||||
-- duplication is the cheaper correctness trade.
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5(
|
||||
title, people, body,
|
||||
tokenize='unicode61 remove_diacritics 2'
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);
|
||||
`;
|
||||
|
||||
export class MailIndexUnavailableError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'MailIndexUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that the file we just opened is REALLY encrypted.
|
||||
*
|
||||
* This is not defensive boilerplate, it guards the sharpest landmine found
|
||||
* while designing this: on both `node:sqlite` and plain `better-sqlite3`,
|
||||
* `PRAGMA key = ...` is **silently accepted and does nothing** - no error, a
|
||||
* working database, and the mail sitting on disk in cleartext. Verified by
|
||||
* writing a file and recovering a canary string from the raw bytes.
|
||||
*
|
||||
* The check is on the VALUE, not the row count: a non-cipher binding returns
|
||||
* ZERO ROWS for `PRAGMA cipher_version`, so a naive `!== ''` comparison over a
|
||||
* missing row passes vacuously. Require a non-empty string.
|
||||
*/
|
||||
function assertEncrypted(db: SqlcipherDatabase, dbPath: string): void {
|
||||
const rows = db.pragma('cipher_version');
|
||||
const value =
|
||||
Array.isArray(rows) && rows.length > 0 && rows[0] && typeof rows[0] === 'object'
|
||||
? (rows[0] as Record<string, unknown>).cipher_version
|
||||
: undefined;
|
||||
if (typeof value !== 'string' || value.trim().length === 0) {
|
||||
db.close();
|
||||
throw new MailIndexUnavailableError(
|
||||
`Refusing to use ${path.basename(dbPath)}: the SQLite binding reports no SQLCipher ` +
|
||||
`support (PRAGMA cipher_version returned ${JSON.stringify(rows)}), so the index ` +
|
||||
`would be written in cleartext.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface OpenOptions {
|
||||
storeDir: string;
|
||||
accountId: string;
|
||||
/** Raw 32-byte key. Used as SQLCipher's raw key (no KDF) via `PRAGMA key = "x'..'"`. */
|
||||
key: Buffer;
|
||||
}
|
||||
|
||||
export class MailIndex {
|
||||
private constructor(
|
||||
private readonly db: SqlcipherDatabase,
|
||||
readonly dbPath: string,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Opens (creating if needed) the account's index. Throws
|
||||
* MailIndexUnavailableError when the native binding is absent or the file is
|
||||
* not actually encrypted; the caller turns the feature off rather than
|
||||
* falling back to something unencrypted.
|
||||
*/
|
||||
static open({ storeDir, accountId, key }: OpenOptions): MailIndex {
|
||||
const Database = loadSqlcipher();
|
||||
if (!Database) {
|
||||
throw new MailIndexUnavailableError(
|
||||
'@signalapp/sqlcipher is not installed for this platform (it is an optional dependency).',
|
||||
);
|
||||
}
|
||||
if (key.length !== 32) {
|
||||
throw new MailIndexUnavailableError(`Index key must be 32 bytes, got ${key.length}.`);
|
||||
}
|
||||
|
||||
const dbPath = indexDbPath(storeDir, accountId);
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: 0o700 });
|
||||
|
||||
let db = new Database(dbPath);
|
||||
// The key pragma must be the FIRST statement on the connection. Hex form
|
||||
// means SQLCipher uses these 32 bytes as the raw key with no KDF, which is
|
||||
// right for a random key (a passphrase would want the KDF).
|
||||
db.pragma(`key = "x'${key.toString('hex')}'"`);
|
||||
assertEncrypted(db, dbPath);
|
||||
|
||||
// A wrong key surfaces here rather than at open: SQLCipher only reads the
|
||||
// header lazily. Treat it as "unreadable" and rebuild from scratch - the
|
||||
// index is derived data, so there is nothing to recover and never anything
|
||||
// to prompt the user for (the key was never a user secret).
|
||||
let version: number | null;
|
||||
try {
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
version = readSchemaVersion(db);
|
||||
} catch {
|
||||
db.close();
|
||||
for (const f of dbSiblings(dbPath)) {
|
||||
try { fs.rmSync(f, { force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
db = new Database(dbPath);
|
||||
db.pragma(`key = "x'${key.toString('hex')}'"`);
|
||||
assertEncrypted(db, dbPath);
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
version = null;
|
||||
}
|
||||
|
||||
if (version !== null && version !== SCHEMA_VERSION) {
|
||||
// Rebuildable derived data: drop, don't migrate.
|
||||
db.exec('DROP TABLE IF EXISTS doc_fts; DROP TABLE IF EXISTS doc; DROP TABLE IF EXISTS meta;');
|
||||
version = null;
|
||||
}
|
||||
if (version === null) {
|
||||
db.exec(DDL);
|
||||
db.prepare('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)').run([
|
||||
'schema_version',
|
||||
String(SCHEMA_VERSION),
|
||||
]);
|
||||
}
|
||||
|
||||
return new MailIndex(db, dbPath);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.db.close(); } catch { /* already closed */ }
|
||||
}
|
||||
|
||||
/**
|
||||
* Upserts documents and keeps the FTS rows in step. Returns the number of
|
||||
* rows written. One transaction for the whole batch - a partially-applied
|
||||
* batch is harmless (it is an index) but a transaction is faster.
|
||||
*/
|
||||
upsert(docs: readonly IndexDoc[]): number {
|
||||
if (docs.length === 0) return 0;
|
||||
|
||||
const upsertDoc = this.db.prepare(`
|
||||
INSERT INTO doc (jmap_account_id, content_type, id, title, people, body,
|
||||
occurred_at, metadata_json, indexed_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(jmap_account_id, content_type, id) DO UPDATE SET
|
||||
title = excluded.title, people = excluded.people, body = excluded.body,
|
||||
occurred_at = excluded.occurred_at, metadata_json = excluded.metadata_json,
|
||||
indexed_at = excluded.indexed_at
|
||||
RETURNING rowid
|
||||
`);
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const insertFts = this.db.prepare(
|
||||
'INSERT INTO doc_fts (rowid, title, people, body) VALUES (?, ?, ?, ?)',
|
||||
);
|
||||
|
||||
const now = Date.now();
|
||||
let written = 0;
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const d of docs) {
|
||||
const row = upsertDoc.get([
|
||||
d.jmapAccountId, d.contentType, d.id,
|
||||
d.title, d.people, d.body,
|
||||
d.occurredAt, JSON.stringify(d.metadata ?? {}), now,
|
||||
]);
|
||||
const rowid = row?.rowid;
|
||||
if (typeof rowid !== 'number') continue;
|
||||
// ON CONFLICT preserves the rowid, so delete-then-insert replaces the
|
||||
// old FTS row rather than accumulating duplicates for one document.
|
||||
deleteFts.run([rowid]);
|
||||
insertFts.run([rowid, d.title, d.people, d.body]);
|
||||
written++;
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
/** Removes documents by id (a JMAP `destroyed` id, or a stale row). */
|
||||
remove(jmapAccountId: string, contentType: ContentType, ids: readonly string[]): number {
|
||||
if (ids.length === 0) return 0;
|
||||
const findRow = this.db.prepare(
|
||||
'SELECT rowid FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
|
||||
);
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const deleteDoc = this.db.prepare(
|
||||
'DELETE FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?',
|
||||
);
|
||||
let removed = 0;
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const row = findRow.get([jmapAccountId, contentType, id]);
|
||||
if (typeof row?.rowid === 'number') deleteFts.run([row.rowid]);
|
||||
removed += deleteDoc.run([jmapAccountId, contentType, id]).changes;
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-text search - the retrieval surface an AI feature calls to gather
|
||||
* context. `types` empty/omitted searches everything.
|
||||
*/
|
||||
search(opts: {
|
||||
query: string;
|
||||
types?: readonly ContentType[];
|
||||
limit?: number;
|
||||
snippetTokens?: number;
|
||||
}): SearchHit[] {
|
||||
const match = toFtsMatchQuery(opts.query);
|
||||
if (!match) return [];
|
||||
|
||||
const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200);
|
||||
const tokens = Math.min(Math.max(opts.snippetTokens ?? 24, 4), 64);
|
||||
const types = opts.types && opts.types.length > 0 ? opts.types : null;
|
||||
const typeFilter = types ? ` AND d.content_type IN (${types.map(() => '?').join(',')})` : '';
|
||||
|
||||
// bm25 weights: a hit in the title or in a name/address is a stronger
|
||||
// signal than one in a long body, and for RAG the title is what makes a
|
||||
// retrieved chunk recognisable.
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people,
|
||||
d.occurred_at, d.metadata_json,
|
||||
bm25(doc_fts, 8.0, 4.0, 1.0) AS score,
|
||||
snippet(doc_fts, 2, '[', ']', '…', ${tokens}) AS snip
|
||||
FROM doc_fts
|
||||
JOIN doc d ON d.rowid = doc_fts.rowid
|
||||
WHERE doc_fts MATCH ?${typeFilter}
|
||||
ORDER BY score ASC, d.occurred_at DESC
|
||||
LIMIT ?
|
||||
`)
|
||||
.all([match, ...(types ?? []), limit]);
|
||||
|
||||
return rows.map((r) => ({
|
||||
contentType: String(r.content_type) as ContentType,
|
||||
id: String(r.id),
|
||||
jmapAccountId: String(r.jmap_account_id),
|
||||
title: String(r.title ?? ''),
|
||||
people: String(r.people ?? ''),
|
||||
occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at),
|
||||
metadata: safeParseObject(r.metadata_json),
|
||||
score: typeof r.score === 'number' ? r.score : 0,
|
||||
snippet: String(r.snip ?? ''),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Per-type counts and freshness, for the Settings UI and for debugging. */
|
||||
stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> {
|
||||
return this.db
|
||||
.prepare(`
|
||||
SELECT content_type, COUNT(*) AS n, MAX(occurred_at) AS newest, MAX(indexed_at) AS indexed
|
||||
FROM doc GROUP BY content_type ORDER BY content_type
|
||||
`)
|
||||
.all()
|
||||
.map((r) => ({
|
||||
contentType: String(r.content_type),
|
||||
count: Number(r.n ?? 0),
|
||||
newest: r.newest === null || r.newest === undefined ? null : String(r.newest),
|
||||
indexedAt: typeof r.indexed === 'number' ? r.indexed : null,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Ids already present, so a catch-up pass can skip re-fetching bodies. */
|
||||
existingIds(jmapAccountId: string, contentType: ContentType): Set<string> {
|
||||
const rows = this.db
|
||||
.prepare('SELECT id FROM doc WHERE jmap_account_id = ? AND content_type = ?')
|
||||
.all([jmapAccountId, contentType]);
|
||||
return new Set(rows.map((r) => String(r.id)));
|
||||
}
|
||||
|
||||
/** Drops documents older than the retention floor for a type. */
|
||||
pruneOlderThan(jmapAccountId: string, contentType: ContentType, isoFloor: string): number {
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT rowid FROM doc
|
||||
WHERE jmap_account_id = ? AND content_type = ?
|
||||
AND occurred_at IS NOT NULL AND occurred_at < ?
|
||||
`)
|
||||
.all([jmapAccountId, contentType, isoFloor]);
|
||||
if (rows.length === 0) return 0;
|
||||
const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?');
|
||||
const deleteDoc = this.db.prepare('DELETE FROM doc WHERE rowid = ?');
|
||||
this.db.exec('BEGIN');
|
||||
try {
|
||||
for (const r of rows) {
|
||||
deleteFts.run([r.rowid]);
|
||||
deleteDoc.run([r.rowid]);
|
||||
}
|
||||
this.db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
}
|
||||
|
||||
function readSchemaVersion(db: SqlcipherDatabase): number | null {
|
||||
try {
|
||||
const row = db.prepare("SELECT v FROM meta WHERE k = 'schema_version'").get();
|
||||
if (!row || row.v === undefined) return null;
|
||||
const n = Number(row.v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
} catch {
|
||||
// `meta` doesn't exist yet - a fresh file.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function safeParseObject(v: unknown): Record<string, unknown> {
|
||||
if (typeof v !== 'string') return {};
|
||||
try {
|
||||
const parsed = JSON.parse(v);
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns arbitrary user text into a safe FTS5 MATCH expression.
|
||||
*
|
||||
* FTS5's query syntax is not SQL, so parameter binding does NOT protect it: a
|
||||
* bare `"` or a stray `*`/`NEAR`/`:` in user input raises
|
||||
* `fts5: syntax error`, which would turn a normal search box into a 500. Every
|
||||
* token is quoted (making it a literal phrase) and a trailing `*` is added to
|
||||
* the last token so typing continues to match as the user types.
|
||||
*
|
||||
* Exported for unit testing - it is the one piece of this file with no
|
||||
* database dependency and the most ways to be wrong.
|
||||
*/
|
||||
export function toFtsMatchQuery(raw: string): string | null {
|
||||
if (typeof raw !== 'string') return null;
|
||||
// Split on anything that isn't a word character or an intra-word mark. Keeps
|
||||
// unicode letters (so "Müller" and "東京" survive) via the u flag.
|
||||
const tokens = raw
|
||||
.normalize('NFC')
|
||||
.split(/[^\p{L}\p{N}_@.'-]+/u)
|
||||
.map((t) => t.replace(/^['-]+|['-]+$/g, ''))
|
||||
.filter((t) => t.length > 0)
|
||||
.slice(0, 24);
|
||||
if (tokens.length === 0) return null;
|
||||
return tokens
|
||||
.map((t, i) => {
|
||||
const quoted = `"${t.replace(/"/g, '""')}"`;
|
||||
// Prefix-match only the final token, and only if it's long enough to not
|
||||
// match half the mailbox.
|
||||
return i === tokens.length - 1 && t.length >= 3 ? `${quoted}*` : quoted;
|
||||
})
|
||||
.join(' AND ');
|
||||
}
|
||||
Reference in New Issue
Block a user