An on-device, SQLCipher-encrypted full-text index the app can retrieve from to
feed an LLM ("prompt against"), for the Electron desktop shell only.
Shape: no persistent background worker and no resident credential. Indexing is
a normal request-scoped API route, triggered by the renderer's EXISTING live
JMAP push connection - so it reacts to each delivery/change rather than polling.
- lib/mail-index/binding.ts guarded require of the optional native binding
- lib/mail-index/paths.ts the VNCMAIL_DESKTOP_STORE_DIR gate + hashed paths
- lib/mail-index/store.ts schema, upsert, FTS5 search, encryption assertion
- lib/mail-index/extract.ts PURE JMAP-object -> document extractors
- lib/mail-index/jmap.ts minimal stateless server-side JMAP client
- lib/mail-index/key.ts per-job key fetch over the inherited fd
- lib/mail-index/reindex.ts the job + slot->account resolution
- electron/key-service.ts safeStorage wrap/unwrap, served over fd 3
- app/api/offline/reindex POST, event-driven + catch-up
- app/api/offline/search GET, the retrieval surface (hits + contextBlock)
- lib/mail-index-client.ts renderer client; StateChange -> index call
- components/settings/local-index-settings.tsx status + manual catch-up
Decisions worth knowing:
* `@signalapp/sqlcipher` is an OPTIONAL dependency with a guarded runtime
require. It publishes six N-API prebuilds and NO build sources, and both
Dockerfiles are node:24-alpine (musl, no matching prebuild) - as a hard
dependency it would break the production image and the integration fixture's
webmail container, neither of which wants this feature.
* Credentials come from the existing per-slot encrypted `jmap_stalwart_ctx`
cookie via lib/stalwart/credentials.ts - the same helper /api/settings and
/api/push/preview already use. It carries a ready-made header for basic AND
bearer accounts, so the indexer never touches the OAuth refresh-token cookie;
a server-side refresh would rotate a token into a response nobody reads and
silently log the user out.
* The encryption key crosses main -> server over an INHERITED FILE DESCRIPTOR,
never an environment variable: env is readable by any process running as the
same OS user, which would defeat using the OS keychain at all. Fetched per
job and zeroed after, so there is no long-lived key copy.
* safeStorage's Linux `basic_text` backend (no keyring) is treated as refusal,
not degradation - it "encrypts" with a hardcoded public password, which would
look like an encrypted mailbox while providing nothing.
getSelectedStorageBackend() is Linux-only and platform-guarded.
* Every store open asserts `PRAGMA cipher_version` returns a non-empty STRING,
not merely a row: a non-cipher binding returns ZERO ROWS, so a row-count check
would pass vacuously while writing the mailbox to disk in cleartext.
* Files are indexed by name/path/date/size only - NOT by extracted content.
Text extraction from arbitrary PDFs/office documents is a separate problem.
* Account-scoped composite keys `(jmap_account_id, content_type, id)` are kept
even though there is one file per account: one login exposes delegated/shared
JMAP accounts too, and JMAP ids are unique only within an account.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
358 lines
13 KiB
TypeScript
358 lines
13 KiB
TypeScript
// A deliberately tiny server-side JMAP client, used only by the indexer.
|
|
//
|
|
// WHY NOT REUSE lib/jmap/client.ts: that class is a 7400-line renderer object.
|
|
// It holds credentials in instance fields, uses `btoa`, opens EventSource /
|
|
// WebSocket push connections, and wires itself into Zustand stores and toast
|
|
// notifications. Importing it into an API route would drag all of that into the
|
|
// server bundle for the sake of four method calls. The existing server-side
|
|
// JMAP code in this repo (lib/auth/verify-jmap-auth.ts) already sets the
|
|
// precedent: plain fetch + an Authorization header.
|
|
//
|
|
// Everything here is stateless - the caller supplies the auth header per call,
|
|
// so there is no resident credential and nothing to invalidate.
|
|
|
|
import type { CalendarEvent, ContactCard, Email, FileNode } from '@/lib/jmap/types';
|
|
|
|
const REQUEST_TIMEOUT_MS = 30_000;
|
|
|
|
export const CAP_CORE = 'urn:ietf:params:jmap:core';
|
|
export const CAP_MAIL = 'urn:ietf:params:jmap:mail';
|
|
export const CAP_CALENDARS = 'urn:ietf:params:jmap:calendars';
|
|
export const CAP_CONTACTS = 'urn:ietf:params:jmap:contacts';
|
|
export const CAP_FILENODE = 'urn:ietf:params:jmap:filenode';
|
|
|
|
export class JmapIndexError extends Error {
|
|
status: number;
|
|
constructor(message: string, status = 502) {
|
|
super(message);
|
|
this.name = 'JmapIndexError';
|
|
this.status = status;
|
|
}
|
|
}
|
|
|
|
export interface JmapSessionInfo {
|
|
apiUrl: string;
|
|
/** Server-confirmed authenticated login (JMAP Session.username). */
|
|
username?: string;
|
|
primaryAccounts: Record<string, string>;
|
|
accounts: Record<string, { name?: string; isPersonal?: boolean; accountCapabilities?: Record<string, unknown> }>;
|
|
capabilities: Record<string, unknown>;
|
|
}
|
|
|
|
/**
|
|
* Pins a URL advertised by the session to the origin we authenticated against.
|
|
*
|
|
* `lib/jmap/client.ts` does the same thing in its rewriteSessionUrls() for the
|
|
* renderer's benefit. Server-side it is a security control, not a convenience:
|
|
* we attach the user's credentials to this URL, so a session document that
|
|
* advertised an `apiUrl` on someone else's host would turn this into a
|
|
* credential-leaking SSRF. Keep the path and query, take the origin from the
|
|
* server URL we were configured with.
|
|
*/
|
|
function pinToServerOrigin(advertised: string, serverUrl: string): string {
|
|
const base = new URL(serverUrl);
|
|
let target: URL;
|
|
try {
|
|
target = new URL(advertised, base);
|
|
} catch {
|
|
throw new JmapIndexError('JMAP session advertised an unusable apiUrl');
|
|
}
|
|
return `${base.origin}${target.pathname}${target.search}`;
|
|
}
|
|
|
|
async function fetchWithTimeout(url: string, init: RequestInit): Promise<Response> {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
|
try {
|
|
return await fetch(url, { ...init, signal: controller.signal, redirect: 'manual' });
|
|
} catch (error) {
|
|
if (error instanceof Error && error.name === 'AbortError') {
|
|
throw new JmapIndexError('JMAP request timed out', 504);
|
|
}
|
|
throw new JmapIndexError(`JMAP request failed: ${String(error)}`);
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
export async function fetchJmapSession(serverUrl: string, authHeader: string): Promise<JmapSessionInfo> {
|
|
const response = await fetchWithTimeout(`${serverUrl.replace(/\/+$/, '')}/.well-known/jmap`, {
|
|
method: 'GET',
|
|
headers: { Authorization: authHeader },
|
|
});
|
|
if (response.status === 401 || response.status === 403) {
|
|
throw new JmapIndexError('JMAP authentication failed', 401);
|
|
}
|
|
if (!response.ok) {
|
|
throw new JmapIndexError(`JMAP session fetch failed (${response.status})`);
|
|
}
|
|
const raw = (await response.json().catch(() => null)) as Record<string, unknown> | null;
|
|
if (!raw || typeof raw.apiUrl !== 'string') {
|
|
throw new JmapIndexError('Invalid JMAP session response');
|
|
}
|
|
return {
|
|
apiUrl: pinToServerOrigin(raw.apiUrl, serverUrl),
|
|
username: typeof raw.username === 'string' ? raw.username : undefined,
|
|
primaryAccounts: (raw.primaryAccounts as Record<string, string>) ?? {},
|
|
accounts: (raw.accounts as JmapSessionInfo['accounts']) ?? {},
|
|
capabilities: (raw.capabilities as Record<string, unknown>) ?? {},
|
|
};
|
|
}
|
|
|
|
type MethodCall = [string, Record<string, unknown>, string];
|
|
|
|
/** Raw method-response tuple, `[name, args, callId]`. `name` is 'error' on failure. */
|
|
type MethodResponse = [string, Record<string, unknown>, string];
|
|
|
|
export async function jmapRequest(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
using: readonly string[],
|
|
methodCalls: readonly MethodCall[],
|
|
): Promise<MethodResponse[]> {
|
|
const response = await fetchWithTimeout(session.apiUrl, {
|
|
method: 'POST',
|
|
headers: { Authorization: authHeader, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ using, methodCalls }),
|
|
});
|
|
if (response.status === 401 || response.status === 403) {
|
|
throw new JmapIndexError('JMAP authentication failed', 401);
|
|
}
|
|
if (response.status === 429) {
|
|
throw new JmapIndexError('JMAP server is rate limiting', 429);
|
|
}
|
|
if (!response.ok) {
|
|
throw new JmapIndexError(`JMAP request failed (${response.status})`);
|
|
}
|
|
const data = (await response.json().catch(() => null)) as { methodResponses?: MethodResponse[] } | null;
|
|
if (!data || !Array.isArray(data.methodResponses)) {
|
|
throw new JmapIndexError('Invalid JMAP response envelope');
|
|
}
|
|
return data.methodResponses;
|
|
}
|
|
|
|
function firstResult(responses: MethodResponse[], expected: string): Record<string, unknown> | null {
|
|
for (const [name, args] of responses) {
|
|
if (name === expected) return args;
|
|
// A method-level error is not fatal for an INDEX: a server that doesn't
|
|
// support one data type should not fail the whole reindex. The caller
|
|
// treats null as "nothing to index for this type".
|
|
if (name === 'error') return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function idsOf(args: Record<string, unknown> | null): string[] {
|
|
const ids = args?.ids;
|
|
return Array.isArray(ids) ? ids.filter((v): v is string => typeof v === 'string') : [];
|
|
}
|
|
|
|
function listOf<T>(args: Record<string, unknown> | null): T[] {
|
|
const list = args?.list;
|
|
return Array.isArray(list) ? (list as T[]) : [];
|
|
}
|
|
|
|
export function accountIdFor(session: JmapSessionInfo, capability: string): string | null {
|
|
const id = session.primaryAccounts[capability];
|
|
return typeof id === 'string' && id.length > 0 ? id : null;
|
|
}
|
|
|
|
export function hasCapability(session: JmapSessionInfo, capability: string): boolean {
|
|
return Object.prototype.hasOwnProperty.call(session.capabilities, capability);
|
|
}
|
|
|
|
/** Per-ACCOUNT capability, mirroring client.ts's supportsFiles() (#563: a server can advertise it while an account has it revoked). */
|
|
export function accountHasCapability(
|
|
session: JmapSessionInfo,
|
|
accountId: string,
|
|
capability: string,
|
|
): boolean {
|
|
const account = session.accounts[accountId];
|
|
if (!account) return false;
|
|
if (account.accountCapabilities && Object.prototype.hasOwnProperty.call(account.accountCapabilities, capability)) {
|
|
return true;
|
|
}
|
|
return account.isPersonal === false;
|
|
}
|
|
|
|
// ── mail ────────────────────────────────────────────────────────────────────
|
|
|
|
/** Properties needed to build a mail IndexDoc. Bodies come via bodyValues. */
|
|
const EMAIL_INDEX_PROPERTIES = [
|
|
'id', 'threadId', 'mailboxIds', 'keywords', 'size', 'receivedAt',
|
|
'from', 'to', 'cc', 'subject', 'preview', 'hasAttachment',
|
|
'textBody', 'htmlBody', 'bodyValues',
|
|
] as const;
|
|
|
|
export async function getEmailsForIndex(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
ids: readonly string[],
|
|
maxBodyBytes: number,
|
|
): Promise<Email[]> {
|
|
if (ids.length === 0) return [];
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
|
['Email/get', {
|
|
accountId,
|
|
ids: [...ids],
|
|
properties: [...EMAIL_INDEX_PROPERTIES],
|
|
// Without these two the bodyValues map comes back EMPTY and every
|
|
// indexed body would silently fall back to `preview`.
|
|
fetchTextBodyValues: true,
|
|
fetchHTMLBodyValues: true,
|
|
maxBodyValueBytes: maxBodyBytes,
|
|
}, 'g'],
|
|
]);
|
|
return listOf<Email>(firstResult(responses, 'Email/get'));
|
|
}
|
|
|
|
export async function queryRecentEmailIds(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
afterIso: string,
|
|
limit: number,
|
|
): Promise<string[]> {
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
|
['Email/query', {
|
|
accountId,
|
|
filter: { after: afterIso },
|
|
sort: [{ property: 'receivedAt', isAscending: false }],
|
|
limit,
|
|
calculateTotal: false,
|
|
}, 'q'],
|
|
]);
|
|
return idsOf(firstResult(responses, 'Email/query'));
|
|
}
|
|
|
|
// ── calendar ────────────────────────────────────────────────────────────────
|
|
|
|
export async function getCalendarEventsForIndex(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
ids: readonly string[],
|
|
): Promise<CalendarEvent[]> {
|
|
if (ids.length === 0) return [];
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
|
['CalendarEvent/get', { accountId, ids: [...ids] }, 'g'],
|
|
]);
|
|
return listOf<CalendarEvent>(firstResult(responses, 'CalendarEvent/get'));
|
|
}
|
|
|
|
export async function queryCalendarEventIds(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
afterIso: string,
|
|
beforeIso: string,
|
|
limit: number,
|
|
): Promise<string[]> {
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [
|
|
['CalendarEvent/query', {
|
|
accountId,
|
|
// LocalDateTime, per the note in lib/jmap/client.ts:307-312 - Stalwart
|
|
// parses these without a zone suffix and ignores unparseable values.
|
|
filter: { after: toLocalDateTime(afterIso), before: toLocalDateTime(beforeIso) },
|
|
limit,
|
|
calculateTotal: false,
|
|
}, 'q'],
|
|
]);
|
|
return idsOf(firstResult(responses, 'CalendarEvent/query'));
|
|
}
|
|
|
|
/** JSCalendar LocalDateTime: `YYYY-MM-DDTHH:MM:SS`, no zone designator. */
|
|
function toLocalDateTime(iso: string): string {
|
|
return iso.replace(/\.\d+/, '').replace(/Z$/, '').slice(0, 19);
|
|
}
|
|
|
|
// ── contacts ────────────────────────────────────────────────────────────────
|
|
|
|
export async function getContactsForIndex(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
ids: readonly string[],
|
|
): Promise<ContactCard[]> {
|
|
if (ids.length === 0) return [];
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
|
['ContactCard/get', { accountId, ids: [...ids] }, 'g'],
|
|
]);
|
|
return listOf<ContactCard>(firstResult(responses, 'ContactCard/get'));
|
|
}
|
|
|
|
export async function queryContactIds(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
limit: number,
|
|
): Promise<string[]> {
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [
|
|
['ContactCard/query', { accountId, limit, calculateTotal: false }, 'q'],
|
|
]);
|
|
return idsOf(firstResult(responses, 'ContactCard/query'));
|
|
}
|
|
|
|
// ── files ───────────────────────────────────────────────────────────────────
|
|
|
|
const FILENODE_INDEX_PROPERTIES = [
|
|
'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified',
|
|
] as const;
|
|
|
|
export async function getFilesForIndex(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
ids: readonly string[],
|
|
): Promise<FileNode[]> {
|
|
if (ids.length === 0) return [];
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
|
['FileNode/get', { accountId, ids: [...ids], properties: [...FILENODE_INDEX_PROPERTIES] }, 'g'],
|
|
]);
|
|
return listOf<FileNode>(firstResult(responses, 'FileNode/get'));
|
|
}
|
|
|
|
export async function queryFileIds(
|
|
session: JmapSessionInfo,
|
|
authHeader: string,
|
|
accountId: string,
|
|
limit: number,
|
|
): Promise<string[]> {
|
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE], [
|
|
['FileNode/query', { accountId, filter: {}, limit, calculateTotal: false }, 'q'],
|
|
]);
|
|
return idsOf(firstResult(responses, 'FileNode/query'));
|
|
}
|
|
|
|
/**
|
|
* Builds `id -> "Parent/Child"` paths for the given nodes, walking `parentId`
|
|
* upward. FileNode only knows its parent, so the caller has to assemble this;
|
|
* unresolvable ancestors just truncate the path rather than failing.
|
|
*/
|
|
export function buildFilePaths(nodes: readonly FileNode[]): Map<string, string> {
|
|
const byId = new Map(nodes.map((n) => [n.id, n]));
|
|
const cache = new Map<string, string>();
|
|
|
|
const resolve = (id: string, depth: number): string => {
|
|
if (depth > 32) return '';
|
|
const cached = cache.get(id);
|
|
if (cached !== undefined) return cached;
|
|
const node = byId.get(id);
|
|
if (!node) return '';
|
|
const parent = node.parentId ? resolve(node.parentId, depth + 1) : '';
|
|
const full = parent ? `${parent}/${node.name}` : node.name;
|
|
cache.set(id, full);
|
|
return full;
|
|
};
|
|
|
|
const out = new Map<string, string>();
|
|
for (const n of nodes) {
|
|
// The document's own `path` metadata is its PARENT directory chain, so a
|
|
// search for "Invoices" matches files inside it without the filename
|
|
// being duplicated into the body.
|
|
out.set(n.id, n.parentId ? resolve(n.parentId, 0) : '');
|
|
}
|
|
return out;
|
|
}
|