diff --git a/app/(main)/[locale]/page.tsx b/app/(main)/[locale]/page.tsx index 00bfd246..3d8250b9 100644 --- a/app/(main)/[locale]/page.tsx +++ b/app/(main)/[locale]/page.tsx @@ -1063,7 +1063,30 @@ export default function Home() { debug.log('push', `[Push] Push notifications enabled for ${cleanups.length} account(s)`); } + // CATCH-UP for the desktop shell's local search index. The index's normal + // trigger is a push StateChange (stores/email-store.ts's handleStateChange), + // but nothing was pushed while the app was closed - and the polling + // transport has no signal for contacts or files at all (client.ts's + // buildStatePollingRequest covers Mailbox/Email/Calendar/CalendarEvent/ + // SieveScript only). So backfill a bounded recent window once per session, + // after push is wired. Fire-and-forget; a no-op outside Electron. + const catchUpTimer = setTimeout(() => { + void (async () => { + try { + const { catchUpIndex } = await import('@/lib/mail-index-client'); + await catchUpIndex( + useAccountStore.getState().getActiveAccount()?.cookieSlot, + ); + } catch { + /* the index is optional */ + } + })(); + // Deliberately after the initial mailbox fetch settles: the catch-up is a + // background nicety and must not compete with first paint. + }, 4000); + return () => { + clearTimeout(catchUpTimer); cleanups.forEach((fn) => fn()); }; }, [isAuthenticated, client, activeAccountId, connectedAccountsSignature, handleStateChange, setPushConnected, buildPopulatedUnifiedAccounts, refreshCrossCounts, refreshUnifiedCounts]); diff --git a/app/api/offline/reindex/route.ts b/app/api/offline/reindex/route.ts new file mode 100644 index 00000000..93ca0843 --- /dev/null +++ b/app/api/offline/reindex/route.ts @@ -0,0 +1,110 @@ +// POST /api/offline/reindex - write mail/calendar/contacts/files into the +// encrypted local search index for the calling session's account. +// +// The PRIMARY caller is the renderer's live JMAP push handler: when a +// StateChange arrives it posts the ids that changed, so indexing is reactive to +// each delivery rather than periodic. `{ catchUp: true }` (no ids) is the +// fallback used at app launch to backfill whatever changed while the app was +// closed. +// +// GATED: returns 404 unless VNCMAIL_DESKTOP_STORE_DIR is set, which only +// electron/main.ts does. The same standalone server artifact runs in the +// multi-tenant production Docker image, where this feature must not exist at +// all - 404 rather than 403 so nothing learns the route is there. +import { NextRequest, NextResponse } from 'next/server'; +import { logger } from '@/lib/logger'; +import { isSqlcipherAvailable } from '@/lib/mail-index/binding'; +import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key'; +import { getStoreDir } from '@/lib/mail-index/paths'; +import { + IndexSessionError, MAX_IDS_PER_CALL, resolveIndexSession, runIndex, + type IndexRequest, +} from '@/lib/mail-index/reindex'; +import { CONTENT_TYPES, isContentType, type ContentType } from '@/lib/mail-index/store'; +import { JmapIndexError } from '@/lib/mail-index/jmap'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +function parseIdMap(raw: unknown): Partial> | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + const out: Partial> = {}; + for (const [key, value] of Object.entries(raw as Record)) { + if (!isContentType(key) || !Array.isArray(value)) continue; + const ids = value + .filter((v): v is string => typeof v === 'string' && v.length > 0 && v.length <= 256) + .slice(0, MAX_IDS_PER_CALL); + if (ids.length > 0) out[key] = ids; + } + return Object.keys(out).length > 0 ? out : undefined; +} + +export async function POST(request: NextRequest) { + if (!getStoreDir()) { + return new NextResponse(null, { status: 404 }); + } + if (!hasKeyChannel()) { + return NextResponse.json( + { error: 'The local index has no key channel in this process.', code: 'no-key-channel' }, + { status: 503 }, + ); + } + if (!isSqlcipherAvailable()) { + // The native binding is an optionalDependency, so "not installed" is a + // normal state on platforms without a prebuild - not an error to log loudly. + return NextResponse.json( + { error: 'Encrypted local index is unavailable on this platform.', code: 'no-binding' }, + { status: 503 }, + ); + } + + let body: Record = {}; + try { + const text = await request.text(); + if (text.trim()) body = JSON.parse(text) as Record; + } catch { + return NextResponse.json({ error: 'Malformed JSON body' }, { status: 400 }); + } + + const rawTypes = Array.isArray(body.types) ? body.types.filter(isContentType) : []; + const req: IndexRequest = { + types: rawTypes.length > 0 ? rawTypes : undefined, + ids: parseIdMap(body.ids), + removed: parseIdMap(body.removed), + // Pruning is a catch-up concern; a single-delivery call shouldn't scan. + prune: body.catchUp === true, + }; + + try { + const session = await resolveIndexSession(request); + const result = await runIndex(session, req); + return NextResponse.json( + { + ok: true, + written: result.written, + skipped: result.skipped, + errors: result.errors, + durationMs: result.durationMs, + types: CONTENT_TYPES, + }, + { headers: { 'Cache-Control': 'no-store' } }, + ); + } catch (error) { + if (error instanceof IndexSessionError) { + return NextResponse.json({ error: error.message }, { status: error.status }); + } + if (error instanceof JmapIndexError) { + return NextResponse.json({ error: error.message }, { status: error.status }); + } + if (error instanceof IndexKeyError) { + // no-secure-storage is the Linux-without-a-keyring refusal: a real, + // expected outcome with a user-facing explanation, not a server fault. + const status = error.code === 'no-secure-storage' ? 503 : 500; + return NextResponse.json({ error: error.message, code: error.code }, { status }); + } + logger.error('mail-index reindex failed', { + error: error instanceof Error ? error.message : String(error), + }); + return NextResponse.json({ error: 'Reindex failed' }, { status: 500 }); + } +} diff --git a/app/api/offline/search/route.ts b/app/api/offline/search/route.ts new file mode 100644 index 00000000..eb938050 --- /dev/null +++ b/app/api/offline/search/route.ts @@ -0,0 +1,117 @@ +// GET /api/offline/search?q=...&types=mail,calendar&limit=20 +// +// THE RETRIEVAL SURFACE. This is what an AI/RAG feature calls to gather +// relevant context from the user's own mail, calendar, contacts and files +// before prompting a model - hence the `snippet` on every hit and the +// `contextBlock` convenience field, which is the same information already +// flattened into text a prompt can carry directly. +// +// Read-only: it never touches the network and never writes. Gated identically +// to the reindex route. +import { NextRequest, NextResponse } from 'next/server'; +import { logger } from '@/lib/logger'; +import { isSqlcipherAvailable } from '@/lib/mail-index/binding'; +import { hasKeyChannel, IndexKeyError, withIndexKey } from '@/lib/mail-index/key'; +import { getStoreDir } from '@/lib/mail-index/paths'; +import { IndexSessionError, resolveIndexSession } from '@/lib/mail-index/reindex'; +import { + isContentType, MailIndex, MailIndexUnavailableError, type ContentType, type SearchHit, +} from '@/lib/mail-index/store'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * One hit as a plain text block, ready to be concatenated into a prompt. + * Kept server-side so every caller (a chat feature, a future agent, a test) + * formats context the same way rather than each inventing its own. + */ +function toContextBlock(hit: SearchHit): string { + const label: Record = { + mail: 'EMAIL', calendar: 'CALENDAR EVENT', contact: 'CONTACT', file: 'FILE', + }; + const lines = [`[${label[hit.contentType]}] ${hit.title}`]; + if (hit.occurredAt) lines.push(`Date: ${hit.occurredAt}`); + if (hit.people) lines.push(`People: ${hit.people}`); + const path = hit.metadata?.path; + if (typeof path === 'string' && path) lines.push(`Path: ${path}`); + if (hit.snippet) lines.push(`Excerpt: ${hit.snippet}`); + return lines.join('\n'); +} + +export async function GET(request: NextRequest) { + if (!getStoreDir()) { + return new NextResponse(null, { status: 404 }); + } + if (!hasKeyChannel() || !isSqlcipherAvailable()) { + return NextResponse.json( + { error: 'Encrypted local index is unavailable in this process.', code: 'unavailable' }, + { status: 503 }, + ); + } + + const params = request.nextUrl.searchParams; + const query = (params.get('q') ?? '').trim(); + const wantStats = params.get('stats') === 'true'; + + if (!query && !wantStats) { + return NextResponse.json({ error: 'Missing q parameter' }, { status: 400 }); + } + if (query.length > 512) { + return NextResponse.json({ error: 'Query too long' }, { status: 400 }); + } + + const types = (params.get('types') ?? '') + .split(',') + .map((t) => t.trim()) + .filter(isContentType); + + const limitRaw = Number(params.get('limit') ?? '20'); + const limit = Number.isFinite(limitRaw) ? Math.min(Math.max(Math.trunc(limitRaw), 1), 100) : 20; + + try { + const session = await resolveIndexSession(request); + const storeDir = getStoreDir(); + if (!storeDir) return new NextResponse(null, { status: 404 }); + + const payload = await withIndexKey(session.accountId, (key) => { + const index = MailIndex.open({ storeDir, accountId: session.accountId, key }); + try { + const stats = index.stats(); + if (!query) return { hits: [] as SearchHit[], stats }; + return { hits: index.search({ query, types, limit }), stats: wantStats ? stats : undefined }; + } finally { + index.close(); + } + }); + + return NextResponse.json( + { + ok: true, + query, + types: types.length > 0 ? types : 'all', + count: payload.hits.length, + hits: payload.hits, + // Everything a prompt needs, pre-joined in rank order. + contextBlock: payload.hits.map(toContextBlock).join('\n\n---\n\n'), + ...(payload.stats ? { stats: payload.stats } : {}), + }, + { headers: { 'Cache-Control': 'no-store' } }, + ); + } catch (error) { + if (error instanceof IndexSessionError) { + return NextResponse.json({ error: error.message }, { status: error.status }); + } + if (error instanceof IndexKeyError) { + const status = error.code === 'no-secure-storage' ? 503 : 500; + return NextResponse.json({ error: error.message, code: error.code }, { status }); + } + if (error instanceof MailIndexUnavailableError) { + return NextResponse.json({ error: error.message, code: 'unavailable' }, { status: 503 }); + } + logger.error('mail-index search failed', { + error: error instanceof Error ? error.message : String(error), + }); + return NextResponse.json({ error: 'Search failed' }, { status: 500 }); + } +} diff --git a/components/settings/about-data-settings.tsx b/components/settings/about-data-settings.tsx index 0c0556b9..b6a333b6 100644 --- a/components/settings/about-data-settings.tsx +++ b/components/settings/about-data-settings.tsx @@ -13,6 +13,7 @@ import { cn } from '@/lib/utils'; import { getPathPrefix } from '@/lib/browser-navigation'; import { clearCachedData } from '@/lib/clear-cached-data'; import { SpamSiegeGame } from './spam-siege-game'; +import { LocalIndexSettings } from './local-index-settings'; const APP_VERSION = process.env.NEXT_PUBLIC_APP_VERSION || "0.0.0"; const GIT_COMMIT = process.env.NEXT_PUBLIC_GIT_COMMIT || "unknown"; @@ -218,6 +219,9 @@ export function AboutDataSettings() { + + {/* Desktop shell only - renders nothing in the browser/PWA build. */} + ); } diff --git a/components/settings/local-index-settings.tsx b/components/settings/local-index-settings.tsx new file mode 100644 index 00000000..670dbeac --- /dev/null +++ b/components/settings/local-index-settings.tsx @@ -0,0 +1,123 @@ +"use client"; + +// Settings panel for the desktop shell's encrypted local search index. +// +// Deliberately small: the index's PRIMARY trigger is the live push connection +// (see lib/mail-index-client.ts's indexOnStateChange, wired into +// stores/email-store.ts's handleStateChange), so this panel is a status readout +// plus a manual catch-up button - not the mechanism. +// +// Renders nothing at all outside the Electron shell, where the routes 404. + +import { useCallback, useEffect, useState } from 'react'; +import { Button } from '@/components/ui/button'; +import { SettingsSection, SettingItem } from './settings-section'; +import { isElectronShell } from '@/lib/electron-bridge'; +import { useAccountStore } from '@/stores/account-store'; +import { catchUpIndex, fetchIndexStats, type IndexStats } from '@/lib/mail-index-client'; + +const TYPE_LABELS: Record = { + mail: 'Mail', + calendar: 'Calendar', + contact: 'Contacts', + file: 'Files', +}; + +export function LocalIndexSettings() { + const slot = useAccountStore((s) => s.accounts.find((a) => a.id === s.activeAccountId)?.cookieSlot); + const [stats, setStats] = useState(null); + const [busy, setBusy] = useState(false); + const [message, setMessage] = useState(null); + // `null` until the first probe resolves, so we don't flash a panel that then + // vanishes on a non-desktop build. + const [available, setAvailable] = useState(null); + + const refreshStats = useCallback(async () => { + const next = await fetchIndexStats(slot); + setStats(next); + setAvailable(next !== null); + }, [slot]); + + useEffect(() => { + if (!isElectronShell()) { + setAvailable(false); + return; + } + void refreshStats(); + }, [refreshStats]); + + const handleRebuild = async () => { + setBusy(true); + setMessage(null); + try { + const result = await catchUpIndex(slot); + if (result.unavailable) { + setAvailable(false); + setMessage(result.error ?? 'The encrypted index is unavailable on this system.'); + return; + } + if (!result.ok) { + setMessage(result.error ?? 'Indexing failed.'); + return; + } + const written = Object.entries(result.written ?? {}) + .map(([type, n]) => `${TYPE_LABELS[type] ?? type}: ${n}`) + .join(', '); + const failed = (result.errors ?? []).map((e) => `${e.contentType} (${e.message})`).join('; '); + setMessage( + [ + written ? `Indexed ${written}.` : 'Nothing to index.', + result.skipped?.length ? `Not supported: ${result.skipped.join(', ')}.` : '', + failed ? `Problems: ${failed}` : '', + ] + .filter(Boolean) + .join(' '), + ); + await refreshStats(); + } finally { + setBusy(false); + } + }; + + if (available === false || available === null) return null; + + const total = (stats ?? []).reduce((sum, s) => sum + s.count, 0); + + return ( + + 0 + ? (stats ?? []) + .map((s) => `${TYPE_LABELS[s.contentType] ?? s.contentType}: ${s.count}`) + .join(' · ') + : 'Nothing indexed yet.' + } + > + {total} + + + + + + + ); +} diff --git a/electron/key-service.ts b/electron/key-service.ts new file mode 100644 index 00000000..3667e91f --- /dev/null +++ b/electron/key-service.ts @@ -0,0 +1,231 @@ +// Main-process key service for the local search index. +// +// The index database is SQLCipher-encrypted with a random per-account 32-byte +// key. That key is wrapped with Electron's `safeStorage` (OS keychain / DPAPI / +// libsecret-or-kwallet) and stored under the store directory. Only the main +// process can call `safeStorage`, but the index itself lives in the standalone +// Next.js server child process - so the unwrapped key has to cross one process +// boundary. +// +// TRANSPORT: an inherited file descriptor (fd 3), NOT an environment variable. +// A nonce or key passed through the spawned process's environment is readable +// by any other process running as the same OS user (`ps eww`, /proc//environ), +// which would defeat the entire point of using the OS keychain. An inherited fd +// is not exposed to process listing. libuv creates extra stdio "pipe" entries +// as socketpairs, so fd 3 is duplex - verified by execution through Electron's +// own spawn before this was built on. +// +// The server side asks for a key only when a reindex job actually runs and drops +// it when the job finishes (see lib/mail-index/key.ts) - there is no long-lived +// resident copy anywhere. +import { safeStorage } from "electron"; +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import type { Readable, Writable } from "node:stream"; + +/** Must match lib/mail-index/paths.ts's accountFileToken(). */ +function accountFileToken(accountId: string): string { + return createHash("sha256").update(accountId, "utf8").digest("hex").slice(0, 32); +} + +function keyFilePath(storeDir: string, accountId: string): string { + return path.join(storeDir, "keys", `${accountFileToken(accountId)}.bin`); +} + +export type KeyServiceFailure = + /** No OS keyring at all - see the Linux note in checkEncryptionAvailable(). */ + | "no-secure-storage" + /** Reading/writing the wrapped key file failed. */ + | "key-io-failed" + /** The wrapped key exists but safeStorage could not decrypt it. */ + | "key-unreadable"; + +export class KeyServiceError extends Error { + code: KeyServiceFailure; + constructor(code: KeyServiceFailure, message: string) { + super(message); + this.name = "KeyServiceError"; + this.code = code; + } +} + +/** + * Decides whether we are willing to store an encryption key on this system. + * + * The Linux caveat is the reason this is a function and not a one-liner: + * `safeStorage.isEncryptionAvailable()` can return **true** on Linux while the + * data is protected by a hardcoded, publicly-known password, with + * `getSelectedStorageBackend()` reporting `basic_text`. That is worse than an + * honest failure, because it looks like it worked. So a `basic_text` backend is + * treated as "no secure storage" and the feature refuses to materialise + * anything - the index is a convenience, and silently pretending a mailbox is + * encrypted when it is not is not a trade worth making. + * + * `getSelectedStorageBackend()` is **Linux-only** and throws elsewhere, hence + * the platform guard. Both calls also require `app.whenReady()`. + */ +export function checkEncryptionAvailable(): { ok: true } | { ok: false; reason: string } { + if (!safeStorage.isEncryptionAvailable()) { + return { ok: false, reason: "The OS reports no secure storage available for encryption keys." }; + } + if (process.platform === "linux") { + let backend: string; + try { + backend = safeStorage.getSelectedStorageBackend(); + } catch { + // Older/newer Electron, or called too early. Be conservative. + return { ok: false, reason: "Could not determine the Linux secret-storage backend." }; + } + if (backend === "basic_text" || backend === "unknown") { + return { + ok: false, + reason: + `No OS keyring is available (backend: ${backend}). Electron would "encrypt" the key ` + + `with a hardcoded password, which provides no real protection, so the encrypted ` + + `local index is disabled on this system.`, + }; + } + } + return { ok: true }; +} + +/** Fetches the account's raw index key, creating and wrapping one on first use. */ +function getOrCreateKey(storeDir: string, accountId: string): Buffer { + const availability = checkEncryptionAvailable(); + if (!availability.ok) { + throw new KeyServiceError("no-secure-storage", availability.reason); + } + + const file = keyFilePath(storeDir, accountId); + + if (fs.existsSync(file)) { + let wrapped: Buffer; + try { + wrapped = fs.readFileSync(file); + } catch (error) { + throw new KeyServiceError("key-io-failed", `Could not read the key file: ${String(error)}`); + } + let hex: string; + try { + hex = safeStorage.decryptString(wrapped); + } catch (error) { + // Most likely cause on macOS: the app's code identity changed (unsigned + // builds get a fresh ad-hoc signature per build), so the Keychain ACL no + // longer matches. Not recoverable and not a user secret - the caller + // deletes the database and re-indexes. + throw new KeyServiceError( + "key-unreadable", + `The stored key could not be decrypted (${String(error)}). It must be recreated.`, + ); + } + const key = Buffer.from(hex.trim(), "hex"); + if (key.length === 32) return key; + // Corrupt payload: fall through and mint a new one. + } + + const key = randomBytes(32); + let wrapped: Buffer; + try { + wrapped = safeStorage.encryptString(key.toString("hex")); + } catch (error) { + throw new KeyServiceError("no-secure-storage", `Could not wrap the key: ${String(error)}`); + } + try { + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + // Write-then-rename so a crash mid-write cannot leave a truncated wrapped + // key that would look like "key-unreadable" forever. + const tmp = `${file}.tmp-${process.pid}`; + fs.writeFileSync(tmp, wrapped, { mode: 0o600 }); + fs.renameSync(tmp, file); + } catch (error) { + throw new KeyServiceError("key-io-failed", `Could not persist the key: ${String(error)}`); + } + return key; +} + +function deleteKey(storeDir: string, accountId: string): void { + try { + fs.rmSync(keyFilePath(storeDir, accountId), { force: true }); + } catch { + /* best effort - the caller is purging anyway */ + } +} + +interface Request { + id?: unknown; + op?: unknown; + accountId?: unknown; +} + +/** + * Serves newline-delimited JSON requests from the standalone server over the + * inherited fd. One line in, one line out, no streaming and no state. + */ +export function attachKeyService( + channel: (Readable & Writable) | null | undefined, + storeDir: string, +): void { + if (!channel) { + console.error("[electron] key service: no channel on fd 3; the local index will be disabled"); + return; + } + + let buffer = ""; + const respond = (payload: Record) => { + try { + channel.write(`${JSON.stringify(payload)}\n`); + } catch (error) { + console.error("[electron] key service: failed to write response:", error); + } + }; + + channel.on("data", (chunk: Buffer | string) => { + buffer += typeof chunk === "string" ? chunk : chunk.toString("utf8"); + // Guard against a peer that never sends a newline. + if (buffer.length > 64 * 1024) buffer = ""; + + let newline: number; + while ((newline = buffer.indexOf("\n")) >= 0) { + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + if (!line.trim()) continue; + + let req: Request; + try { + req = JSON.parse(line) as Request; + } catch { + respond({ id: null, ok: false, code: "bad-request", error: "Malformed request" }); + continue; + } + + const id = typeof req.id === "number" ? req.id : null; + const accountId = typeof req.accountId === "string" ? req.accountId : ""; + if (!accountId) { + respond({ id, ok: false, code: "bad-request", error: "Missing accountId" }); + continue; + } + + try { + if (req.op === "getIndexKey") { + const key = getOrCreateKey(storeDir, accountId); + respond({ id, ok: true, key: key.toString("hex") }); + key.fill(0); + } else if (req.op === "deleteIndexKey") { + deleteKey(storeDir, accountId); + respond({ id, ok: true }); + } else { + respond({ id, ok: false, code: "bad-request", error: `Unknown op: ${String(req.op)}` }); + } + } catch (error) { + const code = error instanceof KeyServiceError ? error.code : "key-io-failed"; + const message = error instanceof Error ? error.message : String(error); + respond({ id, ok: false, code, error: message }); + } + } + }); + + channel.on("error", (error: unknown) => { + console.error("[electron] key service channel error:", error); + }); +} diff --git a/electron/main.ts b/electron/main.ts index aed5c130..bfef6072 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -13,10 +13,26 @@ import { createServer } from "node:net"; import { get as httpGet } from "node:http"; import path from "node:path"; import fs from "node:fs"; +import type { Duplex } from "node:stream"; +import { attachKeyService, checkEncryptionAvailable } from "./key-service"; let serverProcess: ChildProcess | null = null; let mainWindow: BrowserWindow | null = null; +/** + * Root for the encrypted local search index (lib/mail-index/**). Under + * `userData`, so it is per-OS-user and removed with the app's data. + * + * Passing this to the server child process is what ACTIVATES the index: the + * routes 404 without it. That matters because the standalone server is the same + * artifact the production Dockerfile ships to multi-tenant deployments, where a + * server-side index of every user's mail would be badly wrong. One variable + * both enables the feature and supplies its path, so the two cannot drift apart. + */ +function getIndexStoreDir(): string { + return path.join(app.getPath("userData"), "offline"); +} + /** * Locates the standalone server's entrypoint. Packaged builds ship it as an * extraResource (see electron-builder.config.js) because .next/standalone @@ -78,10 +94,29 @@ async function startStandaloneServer(): Promise { const port = await getFreePort(); const url = `http://127.0.0.1:${port}`; + const storeDir = getIndexStoreDir(); + const encryption = checkEncryptionAvailable(); + if (!encryption.ok) { + // Refuse rather than degrade. On Linux with no keyring, safeStorage + // "succeeds" using a hardcoded public password, which would look like an + // encrypted mailbox index while providing no protection. Leaving the env + // vars unset makes every index route 404, so the app runs normally without + // the feature. + console.error(`[electron] local search index disabled: ${encryption.reason}`); + } + // Spawn the Electron binary itself as a plain Node process // (ELECTRON_RUN_AS_NODE) instead of depending on a system Node install - // the packaged app can't assume Node exists on the target machine, and // this keeps dev/packaged behavior identical. + // + // stdio gains a 4th entry: fd 3 is the key channel for the local index (see + // electron/key-service.ts). libuv creates extra stdio "pipe" entries as + // socketpairs, so it is duplex in both directions - verified by execution + // before this was built on. Deliberately NOT an environment variable: env is + // readable by any process running as the same OS user, which would defeat + // using the OS keychain at all. The fd NUMBER below is not a secret; only + // what travels over it is. serverProcess = spawn(process.execPath, [serverEntry], { env: { ...process.env, @@ -89,10 +124,19 @@ async function startStandaloneServer(): Promise { PORT: String(port), HOSTNAME: "127.0.0.1", NODE_ENV: process.env.NODE_ENV || "production", + ...(encryption.ok + ? { VNCMAIL_DESKTOP_STORE_DIR: storeDir, VNCMAIL_DESKTOP_KEY_FD: "3" } + : {}), }, - stdio: "inherit", + stdio: encryption.ok + ? ["inherit", "inherit", "inherit", "pipe"] + : "inherit", }); + if (encryption.ok) { + attachKeyService(serverProcess.stdio[3] as Duplex | null, storeDir); + } + serverProcess.on("exit", (code, signal) => { if (code !== 0 && code !== null) { console.error(`[electron] standalone server exited early (code=${code}, signal=${signal})`); diff --git a/lib/mail-index-client.ts b/lib/mail-index-client.ts new file mode 100644 index 00000000..2a1e09d2 --- /dev/null +++ b/lib/mail-index-client.ts @@ -0,0 +1,199 @@ +// Renderer-side client for the encrypted local search index. +// +// The index is EVENT-DRIVEN: the renderer already holds the live JMAP push +// connection (WebSocket -> SSE -> polling, `lib/jmap/client.ts`'s +// setupPushNotifications), so the moment a StateChange announces new mail, a +// calendar change, a contact edit or a file upload, this posts to the reindex +// route. No polling loop, no background worker, no long-lived credential - +// just one more authenticated fetch from the place the push already arrives. +// +// Every function here is best-effort and never throws: a search index failing +// to update must never break the mail UI. + +import { apiFetch } from '@/lib/browser-navigation'; +import { debug } from '@/lib/debug'; +import type { StateChange } from '@/lib/jmap/types'; + +export type IndexContentType = 'mail' | 'calendar' | 'contact' | 'file'; + +export interface IndexRunResult { + ok: boolean; + written?: Partial>; + skipped?: IndexContentType[]; + errors?: Array<{ contentType: IndexContentType; message: string }>; + durationMs?: number; + /** Set when the feature isn't available (not the desktop shell, no keyring, no binding). */ + unavailable?: boolean; + error?: string; +} + +/** + * Maps JMAP `StateChange` type keys onto our content types. + * + * The transport is already type-generic - the WebSocket handler + * (`client.ts:6308-6316`) and the SSE handler (`:6505`) pass the whole + * `changed` map through untouched, and the WS subscribes with + * `dataTypes: null` (every type) - so anything the server pushes arrives here. + * + * `Mailbox` is deliberately NOT mapped: a Mailbox state change is usually just + * an unread-count move, and it fires constantly. `Email` covers the cases that + * change indexable content. + */ +const STATE_TYPE_TO_CONTENT: Record = { + Email: 'mail', + Calendar: 'calendar', + CalendarEvent: 'calendar', + ContactCard: 'contact', + AddressBook: 'contact', + FileNode: 'file', +}; + +export function contentTypesFromStateChange(change: StateChange): IndexContentType[] { + const out = new Set(); + for (const perAccount of Object.values(change.changed ?? {})) { + for (const stateType of Object.keys(perAccount ?? {})) { + const mapped = STATE_TYPE_TO_CONTENT[stateType]; + if (mapped) out.add(mapped); + } + } + return [...out]; +} + +export interface IndexRequestOptions { + types?: readonly IndexContentType[]; + /** + * Per-type ids to index. Supply them whenever the renderer already knows + * which objects changed - it turns the call into a couple of `Foo/get`s + * instead of a windowed query. Mail is the frequent case and the one where + * this matters. + */ + ids?: Partial>; + /** Backfill the recent window for every supported type, and prune. */ + catchUp?: boolean; + /** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */ + slot?: number; +} + +let inFlight: Promise | null = null; +/** Set once the server says the feature isn't there, so we stop asking. */ +let knownUnavailable = false; + +/** + * Posts one index request. Single-flighted: a burst of deliveries coalesces + * into the in-flight call rather than queueing N overlapping SQLite writers. + */ +export async function requestIndex(options: IndexRequestOptions = {}): Promise { + if (knownUnavailable) return { ok: false, unavailable: true }; + if (inFlight) return inFlight; + + const query = typeof options.slot === 'number' ? `?slot=${options.slot}` : ''; + const run = (async (): Promise => { + try { + const response = await apiFetch(`/api/offline/reindex${query}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + types: options.types, + ids: options.ids, + catchUp: options.catchUp === true, + }), + }); + + // 404 = not the desktop shell (or the feature is gated off). Permanent for + // this page load; stop asking so a busy mailbox doesn't post per delivery. + if (response.status === 404) { + knownUnavailable = true; + return { ok: false, unavailable: true }; + } + if (response.status === 503) { + // No keyring / no native binding / no key channel. Also permanent for + // this session, and the message is worth surfacing in Settings. + knownUnavailable = true; + const body = await response.json().catch(() => ({})); + return { ok: false, unavailable: true, error: body?.error }; + } + if (!response.ok) { + const body = await response.json().catch(() => ({})); + return { ok: false, error: body?.error || `HTTP ${response.status}` }; + } + const body = await response.json(); + debug.log('push', '[index] reindex done', body?.written, body?.errors); + return { + ok: true, + written: body?.written, + skipped: body?.skipped, + errors: body?.errors, + durationMs: body?.durationMs, + }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + inFlight = null; + } + })(); + + inFlight = run; + return run; +} + +/** + * The event-driven entry point, called from the push handler. + * + * `mailIds` lets the caller hand over the ids it already has (the refreshed + * mailbox page), so the frequent mail case costs one `Email/get` rather than a + * 30-day query. The other three types are rare events (a contact edit, a file + * upload, a calendar change), so they fall back to their own bounded queries. + */ +export function indexOnStateChange( + change: StateChange, + opts: { mailIds?: string[]; slot?: number } = {}, +): void { + if (knownUnavailable) return; + const types = contentTypesFromStateChange(change); + if (types.length === 0) return; + + const ids: Partial> = {}; + if (types.includes('mail') && opts.mailIds && opts.mailIds.length > 0) { + ids.mail = opts.mailIds.slice(0, 100); + } + + // Fire-and-forget on purpose: this runs inside the push handler, and the mail + // UI must not wait on a search index. + void requestIndex({ types, ids: Object.keys(ids).length > 0 ? ids : undefined, slot: opts.slot }); +} + +/** + * Launch-time catch-up: backfills whatever changed while the app was closed, + * for which no push event was ever delivered. Also the recovery path for the + * polling transport, which has no signal for contacts or files at all + * (`client.ts`'s buildStatePollingRequest polls Mailbox/Email/Calendar/ + * CalendarEvent/SieveScript only). + */ +export async function catchUpIndex(slot?: number): Promise { + return requestIndex({ catchUp: true, slot }); +} + +export interface IndexStats { + contentType: string; + count: number; + newest: string | null; + indexedAt: number | null; +} + +/** Reads per-type counts without searching. Used by the Settings panel. */ +export async function fetchIndexStats(slot?: number): Promise { + const slotQuery = typeof slot === 'number' ? `&slot=${slot}` : ''; + try { + const response = await apiFetch(`/api/offline/search?stats=true${slotQuery}`); + if (!response.ok) return null; + const body = await response.json(); + return Array.isArray(body?.stats) ? (body.stats as IndexStats[]) : []; + } catch { + return null; + } +} + +/** Resets the "don't ask again" latch - e.g. after the user signs in again. */ +export function resetIndexAvailability(): void { + knownUnavailable = false; +} diff --git a/lib/mail-index/binding.ts b/lib/mail-index/binding.ts new file mode 100644 index 00000000..95227018 --- /dev/null +++ b/lib/mail-index/binding.ts @@ -0,0 +1,83 @@ +// Guarded loader for the SQLCipher native binding. +// +// WHY THIS FILE EXISTS AT ALL: `@signalapp/sqlcipher` is declared in +// package.json's `optionalDependencies`, not `dependencies`, and it MUST stay +// there. It publishes six N-API prebuilds (darwin/linux/win32 x arm64/x64) and +// **no build sources at all** - the published tarball has no `binding.gyp`, no +// `src/`, no `deps/`. Its install script is `node-gyp-build`, which falls back +// to `node-gyp rebuild` when no prebuild matches, and that fallback cannot +// succeed without sources. So on a platform with no matching prebuild the +// install FAILS. +// +// Both Dockerfiles in this repo are `FROM node:24-alpine` + `npm ci` +// (`Dockerfile:1-4`, `integration/webmail.Dockerfile:15-19`). Alpine is musl; +// there is no `linuxmusl-*` prebuild (and the glibc prebuild could not load +// there anyway). As a hard `dependencies` entry this would break the +// production image build and the integration fixture's webmail container - +// neither of which wants this feature, they just need `npm ci` to exit 0. +// `optionalDependencies` makes npm treat that install failure as non-fatal and +// simply omit the package. +// +// The cost of that choice is exactly this module: the require must be guarded +// at runtime, because "installed" is no longer guaranteed. Callers get +// `null` and the feature turns itself off, which is the correct behaviour for +// a desktop-only search index in a server that may not be a desktop. + +/** + * Minimal structural type for the bits of `@signalapp/sqlcipher` we use. + * + * Deliberately hand-written rather than `typeof import('@signalapp/sqlcipher')`: + * the package is optional, so a type-only import would make `tsc` fail on any + * machine where the install was skipped - which is every Alpine CI container. + * + * NOTE the parameter shape. `@signalapp/sqlcipher` is NOT drop-in compatible + * with better-sqlite3 here: its `#checkParams` throws + * `TypeError: Params must be either object or array`, so `stmt.run(a, b, c)` + * (varargs, which better-sqlite3 accepts) is a runtime error. Always pass a + * single array or object. Found by executing it, not by reading the types. + */ +export interface SqlcipherStatement { + run(params?: readonly unknown[] | Record): { changes: number; lastInsertRowid: number }; + get(params?: readonly unknown[] | Record): Record | undefined; + all(params?: readonly unknown[] | Record): Array>; +} + +export interface SqlcipherDatabase { + exec(sql: string): void; + prepare(sql: string): SqlcipherStatement; + pragma(source: string): unknown; + close(): void; +} + +export interface SqlcipherConstructor { + new (path?: string): SqlcipherDatabase; +} + +let cached: SqlcipherConstructor | null | undefined; + +/** + * Returns the Database constructor, or `null` when the optional native binding + * is not installed / cannot load on this platform. Never throws. + * + * Memoised on both outcomes so a missing binding costs one failed require per + * process rather than one per request. + */ +export function loadSqlcipher(): SqlcipherConstructor | null { + if (cached !== undefined) return cached; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const mod = require('@signalapp/sqlcipher') as + | { default?: SqlcipherConstructor } + | SqlcipherConstructor; + const ctor = (mod as { default?: SqlcipherConstructor }).default ?? (mod as SqlcipherConstructor); + cached = typeof ctor === 'function' ? ctor : null; + } catch { + cached = null; + } + return cached; +} + +/** True when the local index can work at all in this process. */ +export function isSqlcipherAvailable(): boolean { + return loadSqlcipher() !== null; +} diff --git a/lib/mail-index/extract.ts b/lib/mail-index/extract.ts new file mode 100644 index 00000000..b209bb1c --- /dev/null +++ b/lib/mail-index/extract.ts @@ -0,0 +1,311 @@ +// PURE JMAP-object -> IndexDoc extractors. +// +// Deliberately free of database, network and store access so every shape +// decision here is unit-testable on its own. The JMAP shapes are awkward +// enough (JSContact keyed maps, JSCalendar participants, FileNode's `modified` +// rather than `updated`) that this is where the bugs would otherwise hide. + +import type { Email, CalendarEvent, ContactCard, FileNode, EmailAddress } from '@/lib/jmap/types'; +import type { IndexDoc } from './store'; + +/** Hard cap on indexed body text per document. Keeps one enormous mail from dominating the file. */ +export const MAX_BODY_CHARS = 32_000; + +/** + * Minimal HTML -> text, for mail that has no `text/plain` alternative. + * + * Not a sanitiser and not trying to be: this output is never rendered, only + * tokenised by FTS5 and possibly handed to an LLM as context. The repo's + * `dompurify` needs a DOM and this runs in Node, so a DOM-free reduction is the + * right tool. Order matters - script/style content must go before tags are + * stripped, or their contents would leak into the index as searchable text. + */ +export function htmlToText(html: string): string { + return html + .replace(//g, ' ') + .replace(/<(script|style|head)\b[\s\S]*?<\/\1>/gi, ' ') + .replace(//gi, '\n') + .replace(/<\/(p|div|tr|li|h[1-6]|blockquote)>/gi, '\n') + .replace(/<[^>]+>/g, ' ') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/&#(\d+);/g, (_m, d: string) => { + const code = Number(d); + return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' '; + }) + .replace(/&#x([0-9a-f]+);/gi, (_m, h: string) => { + const code = parseInt(h, 16); + return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : ' '; + }) + .replace(/[ \t\u00a0]+/g, ' ') + .replace(/\s*\n\s*/g, '\n') + .trim(); +} + +export function normaliseText(s: string | null | undefined): string { + if (!s) return ''; + return s.replace(/\r\n?/g, '\n').replace(/[ \t\u00a0]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim(); +} + +function clamp(s: string, max = MAX_BODY_CHARS): string { + return s.length <= max ? s : s.slice(0, max); +} + +function formatAddresses(list: readonly EmailAddress[] | undefined): string { + if (!list || list.length === 0) return ''; + return list + .map((a) => [a.name, a.email].filter((p) => typeof p === 'string' && p.length > 0).join(' ')) + .filter((s) => s.length > 0) + .join(', '); +} + +/** Values of a JSContact/JSCalendar keyed map, in a stable order. */ +function mapValues(m: Record | null | undefined): T[] { + if (!m || typeof m !== 'object') return []; + return Object.keys(m).sort().map((k) => m[k]); +} + +function joinUnique(parts: Array): string { + const seen = new Set(); + const out: string[] = []; + for (const p of parts) { + const v = typeof p === 'string' ? p.trim() : ''; + if (!v || seen.has(v)) continue; + seen.add(v); + out.push(v); + } + return out.join(', '); +} + +// ── mail ──────────────────────────────────────────────────────────────────── + +/** + * Resolves an Email's plain-text body from `bodyValues`, preferring the + * `text/plain` alternative and falling back to flattening the HTML one. + * + * `textBody`/`htmlBody` reference parts by `partId`; the text itself only + * arrives in `bodyValues` when the `Email/get` asked for it + * (`fetchTextBodyValues` / `fetchHTMLBodyValues`). A caller that forgets that + * gets an empty body rather than an error, which is exactly the kind of silent + * hole worth naming here. + */ +export function emailBodyText(email: Email): string { + const values = email.bodyValues ?? {}; + const fromParts = (parts: typeof email.textBody): string => + (parts ?? []) + .map((p) => values[p.partId]?.value ?? '') + .filter((v) => v.length > 0) + .join('\n\n'); + + const plain = fromParts(email.textBody); + if (plain.trim().length > 0) return normaliseText(plain); + + const html = fromParts(email.htmlBody); + if (html.trim().length > 0) return normaliseText(htmlToText(html)); + + // Last resort: the server-computed preview. Better than nothing for a search + // index, and it costs no extra round trip. + return normaliseText(email.preview); +} + +export function extractMail(jmapAccountId: string, email: Email): IndexDoc { + const body = clamp(emailBodyText(email)); + return { + jmapAccountId, + contentType: 'mail', + id: email.id, + title: normaliseText(email.subject) || '(no subject)', + people: joinUnique([ + formatAddresses(email.from), + formatAddresses(email.to), + formatAddresses(email.cc), + ]), + body, + occurredAt: email.receivedAt ?? null, + metadata: { + threadId: email.threadId, + from: email.from?.[0]?.email ?? null, + fromName: email.from?.[0]?.name ?? null, + hasAttachment: !!email.hasAttachment, + size: email.size ?? null, + mailboxIds: Object.keys(email.mailboxIds ?? {}), + preview: normaliseText(email.preview).slice(0, 300), + }, + }; +} + +// ── calendar ──────────────────────────────────────────────────────────────── + +export function extractCalendarEvent(jmapAccountId: string, event: CalendarEvent): IndexDoc { + const participants = mapValues(event.participants); + const participantText = joinUnique( + participants.flatMap((p) => [ + p?.name, + p?.email, + p?.calendarAddress?.replace(/^mailto:/i, ''), + ...Object.values(p?.sendTo ?? {}).map((v) => + typeof v === 'string' ? v.replace(/^mailto:/i, '') : '', + ), + ]), + ); + + const locations = mapValues(event.locations) + .map((l) => normaliseText(l?.name)) + .filter((s) => s.length > 0); + + // `descriptionContentType` can legitimately be text/html. + const rawDescription = normaliseText(event.description); + const description = /html/i.test(event.descriptionContentType ?? '') + ? normaliseText(htmlToText(rawDescription)) + : rawDescription; + + const keywords = Object.keys(event.keywords ?? {}); + const categories = Object.keys(event.categories ?? {}); + + return { + jmapAccountId, + contentType: 'calendar', + id: event.id, + title: normaliseText(event.title) || '(untitled event)', + people: joinUnique([event.organizerCalendarAddress?.replace(/^mailto:/i, ''), participantText]), + body: clamp( + [description, locations.join(', '), keywords.join(' '), categories.join(' ')] + .filter((s) => s.length > 0) + .join('\n\n'), + ), + // `utcStart` is the resolved instant the app computes; `start` is local + // wall-clock without a zone, so prefer utcStart for ordering. + occurredAt: event.utcStart ?? event.start ?? null, + metadata: { + start: event.start ?? null, + utcStart: event.utcStart ?? null, + utcEnd: event.utcEnd ?? null, + timeZone: event.timeZone ?? null, + showWithoutTime: !!event.showWithoutTime, + status: event.status ?? null, + locations, + calendarIds: Object.keys(event.calendarIds ?? {}), + participantCount: participants.length, + }, + }; +} + +// ── contacts ──────────────────────────────────────────────────────────────── + +export function contactDisplayName(card: ContactCard): string { + const full = normaliseText(card.name?.full); + if (full) return full; + const components = card.name?.components ?? []; + const ordered = ['prefix', 'given', 'given2', 'additional', 'middle', 'surname', 'surname2', 'suffix']; + const byKind = components + .slice() + .sort((a, b) => ordered.indexOf(a.kind) - ordered.indexOf(b.kind)) + .map((c) => c.value) + .filter((v) => typeof v === 'string' && v.trim().length > 0) + .join(' '); + if (byKind.trim()) return normaliseText(byKind); + const firstEmail = mapValues(card.emails)[0]?.address; + if (firstEmail) return firstEmail; + const org = mapValues(card.organizations)[0]?.name; + return normaliseText(org) || '(unnamed contact)'; +} + +export function extractContact(jmapAccountId: string, card: ContactCard): IndexDoc { + const emails = mapValues(card.emails).map((e) => e.address).filter(Boolean); + const phones = mapValues(card.phones).map((p) => p.number).filter(Boolean); + const nicknames = mapValues(card.nicknames) + .map((n) => n?.name) + .filter((v): v is string => typeof v === 'string' && v.length > 0); + const orgs = mapValues(card.organizations).map((o) => o.name).filter((v): v is string => !!v); + const titles = mapValues(card.titles).map((t) => t.name).filter(Boolean); + const notes = mapValues(card.notes).map((n) => n.note).filter(Boolean); + // `full` (RFC 9553) when present, else the legacy flat fields vCard import + // produces, else the ordered components. All three shapes occur in this type. + const addresses = mapValues(card.addresses) + .map((a) => + normaliseText( + a?.full || + [a?.street, a?.locality, a?.region, a?.postcode, a?.country] + .filter((p): p is string => typeof p === 'string' && p.length > 0) + .join(', ') || + (a?.components ?? []).map((c) => c.value).join(' '), + ), + ) + .filter((s) => s.length > 0); + + return { + jmapAccountId, + contentType: 'contact', + id: card.id, + title: contactDisplayName(card), + // Emails/phones go in `people` (weighted above body) because "who is + // this / what's their number" is the dominant contact lookup. + people: joinUnique([...emails, ...phones, ...nicknames]), + body: clamp([...orgs, ...titles, ...addresses, ...notes].filter(Boolean).join('\n')), + // A contact has no meaningful single date; JSContact `updated` is optional + // and not on this repo's type, so leave it null and rank by relevance only. + occurredAt: null, + metadata: { + kind: card.kind ?? null, + emails, + phones, + organizations: orgs, + addressBookIds: Object.keys(card.addressBookIds ?? {}), + }, + }; +} + +// ── files ─────────────────────────────────────────────────────────────────── + +/** + * METADATA ONLY - filename, path, dates, size, owner. Deliberately NOT file + * content: extracting searchable text from arbitrary PDFs / office documents / + * images is a materially bigger problem (per-format parsers, OCR, size limits, + * untrusted-input parsing in a process holding the user's mail) and is a + * separate piece of work. `path` is passed in by the caller because a FileNode + * only knows its `parentId`; resolving the chain is the caller's job. + */ +export function extractFile( + jmapAccountId: string, + node: FileNode, + opts: { path?: string; ownerName?: string } = {}, +): IndexDoc { + const dirPath = normaliseText(opts.path); + const isDirectory = node.type === 'd'; + return { + jmapAccountId, + contentType: 'file', + id: node.id, + title: normaliseText(node.name) || '(unnamed file)', + people: joinUnique([opts.ownerName, node.accountName]), + // The path is genuinely searchable text ("that thing in Invoices/2026"), + // and the extension is worth tokenising on its own. + body: clamp( + [dirPath, isDirectory ? 'folder' : node.type, fileExtension(node.name)] + .filter((s) => s && s.length > 0) + .join('\n'), + ), + // FileNode has `modified`, NOT `updated` - asking for the wrong name + // silently yields undefined (this repo hit that as #700). + occurredAt: node.modified ?? node.created ?? null, + metadata: { + path: dirPath || null, + mimeType: isDirectory ? null : node.type, + isDirectory, + size: typeof node.size === 'number' ? node.size : null, + created: node.created ?? null, + modified: node.modified ?? null, + parentId: node.parentId ?? null, + contentIndexed: false, + }, + }; +} + +function fileExtension(name: string | undefined): string { + if (!name) return ''; + const i = name.lastIndexOf('.'); + return i > 0 && i < name.length - 1 ? name.slice(i + 1).toLowerCase() : ''; +} diff --git a/lib/mail-index/jmap.ts b/lib/mail-index/jmap.ts new file mode 100644 index 00000000..c918d2d6 --- /dev/null +++ b/lib/mail-index/jmap.ts @@ -0,0 +1,357 @@ +// A deliberately tiny server-side JMAP client, used only by the indexer. +// +// WHY NOT REUSE lib/jmap/client.ts: that class is a 7400-line renderer object. +// It holds credentials in instance fields, uses `btoa`, opens EventSource / +// WebSocket push connections, and wires itself into Zustand stores and toast +// notifications. Importing it into an API route would drag all of that into the +// server bundle for the sake of four method calls. The existing server-side +// JMAP code in this repo (lib/auth/verify-jmap-auth.ts) already sets the +// precedent: plain fetch + an Authorization header. +// +// Everything here is stateless - the caller supplies the auth header per call, +// so there is no resident credential and nothing to invalidate. + +import type { CalendarEvent, ContactCard, Email, FileNode } from '@/lib/jmap/types'; + +const REQUEST_TIMEOUT_MS = 30_000; + +export const CAP_CORE = 'urn:ietf:params:jmap:core'; +export const CAP_MAIL = 'urn:ietf:params:jmap:mail'; +export const CAP_CALENDARS = 'urn:ietf:params:jmap:calendars'; +export const CAP_CONTACTS = 'urn:ietf:params:jmap:contacts'; +export const CAP_FILENODE = 'urn:ietf:params:jmap:filenode'; + +export class JmapIndexError extends Error { + status: number; + constructor(message: string, status = 502) { + super(message); + this.name = 'JmapIndexError'; + this.status = status; + } +} + +export interface JmapSessionInfo { + apiUrl: string; + /** Server-confirmed authenticated login (JMAP Session.username). */ + username?: string; + primaryAccounts: Record; + accounts: Record }>; + capabilities: Record; +} + +/** + * Pins a URL advertised by the session to the origin we authenticated against. + * + * `lib/jmap/client.ts` does the same thing in its rewriteSessionUrls() for the + * renderer's benefit. Server-side it is a security control, not a convenience: + * we attach the user's credentials to this URL, so a session document that + * advertised an `apiUrl` on someone else's host would turn this into a + * credential-leaking SSRF. Keep the path and query, take the origin from the + * server URL we were configured with. + */ +function pinToServerOrigin(advertised: string, serverUrl: string): string { + const base = new URL(serverUrl); + let target: URL; + try { + target = new URL(advertised, base); + } catch { + throw new JmapIndexError('JMAP session advertised an unusable apiUrl'); + } + return `${base.origin}${target.pathname}${target.search}`; +} + +async function fetchWithTimeout(url: string, init: RequestInit): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + try { + return await fetch(url, { ...init, signal: controller.signal, redirect: 'manual' }); + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + throw new JmapIndexError('JMAP request timed out', 504); + } + throw new JmapIndexError(`JMAP request failed: ${String(error)}`); + } finally { + clearTimeout(timer); + } +} + +export async function fetchJmapSession(serverUrl: string, authHeader: string): Promise { + const response = await fetchWithTimeout(`${serverUrl.replace(/\/+$/, '')}/.well-known/jmap`, { + method: 'GET', + headers: { Authorization: authHeader }, + }); + if (response.status === 401 || response.status === 403) { + throw new JmapIndexError('JMAP authentication failed', 401); + } + if (!response.ok) { + throw new JmapIndexError(`JMAP session fetch failed (${response.status})`); + } + const raw = (await response.json().catch(() => null)) as Record | null; + if (!raw || typeof raw.apiUrl !== 'string') { + throw new JmapIndexError('Invalid JMAP session response'); + } + return { + apiUrl: pinToServerOrigin(raw.apiUrl, serverUrl), + username: typeof raw.username === 'string' ? raw.username : undefined, + primaryAccounts: (raw.primaryAccounts as Record) ?? {}, + accounts: (raw.accounts as JmapSessionInfo['accounts']) ?? {}, + capabilities: (raw.capabilities as Record) ?? {}, + }; +} + +type MethodCall = [string, Record, string]; + +/** Raw method-response tuple, `[name, args, callId]`. `name` is 'error' on failure. */ +type MethodResponse = [string, Record, string]; + +export async function jmapRequest( + session: JmapSessionInfo, + authHeader: string, + using: readonly string[], + methodCalls: readonly MethodCall[], +): Promise { + const response = await fetchWithTimeout(session.apiUrl, { + method: 'POST', + headers: { Authorization: authHeader, 'Content-Type': 'application/json' }, + body: JSON.stringify({ using, methodCalls }), + }); + if (response.status === 401 || response.status === 403) { + throw new JmapIndexError('JMAP authentication failed', 401); + } + if (response.status === 429) { + throw new JmapIndexError('JMAP server is rate limiting', 429); + } + if (!response.ok) { + throw new JmapIndexError(`JMAP request failed (${response.status})`); + } + const data = (await response.json().catch(() => null)) as { methodResponses?: MethodResponse[] } | null; + if (!data || !Array.isArray(data.methodResponses)) { + throw new JmapIndexError('Invalid JMAP response envelope'); + } + return data.methodResponses; +} + +function firstResult(responses: MethodResponse[], expected: string): Record | null { + for (const [name, args] of responses) { + if (name === expected) return args; + // A method-level error is not fatal for an INDEX: a server that doesn't + // support one data type should not fail the whole reindex. The caller + // treats null as "nothing to index for this type". + if (name === 'error') return null; + } + return null; +} + +function idsOf(args: Record | null): string[] { + const ids = args?.ids; + return Array.isArray(ids) ? ids.filter((v): v is string => typeof v === 'string') : []; +} + +function listOf(args: Record | null): T[] { + const list = args?.list; + return Array.isArray(list) ? (list as T[]) : []; +} + +export function accountIdFor(session: JmapSessionInfo, capability: string): string | null { + const id = session.primaryAccounts[capability]; + return typeof id === 'string' && id.length > 0 ? id : null; +} + +export function hasCapability(session: JmapSessionInfo, capability: string): boolean { + return Object.prototype.hasOwnProperty.call(session.capabilities, capability); +} + +/** Per-ACCOUNT capability, mirroring client.ts's supportsFiles() (#563: a server can advertise it while an account has it revoked). */ +export function accountHasCapability( + session: JmapSessionInfo, + accountId: string, + capability: string, +): boolean { + const account = session.accounts[accountId]; + if (!account) return false; + if (account.accountCapabilities && Object.prototype.hasOwnProperty.call(account.accountCapabilities, capability)) { + return true; + } + return account.isPersonal === false; +} + +// ── mail ──────────────────────────────────────────────────────────────────── + +/** Properties needed to build a mail IndexDoc. Bodies come via bodyValues. */ +const EMAIL_INDEX_PROPERTIES = [ + 'id', 'threadId', 'mailboxIds', 'keywords', 'size', 'receivedAt', + 'from', 'to', 'cc', 'subject', 'preview', 'hasAttachment', + 'textBody', 'htmlBody', 'bodyValues', +] as const; + +export async function getEmailsForIndex( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + ids: readonly string[], + maxBodyBytes: number, +): Promise { + if (ids.length === 0) return []; + const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [ + ['Email/get', { + accountId, + ids: [...ids], + properties: [...EMAIL_INDEX_PROPERTIES], + // Without these two the bodyValues map comes back EMPTY and every + // indexed body would silently fall back to `preview`. + fetchTextBodyValues: true, + fetchHTMLBodyValues: true, + maxBodyValueBytes: maxBodyBytes, + }, 'g'], + ]); + return listOf(firstResult(responses, 'Email/get')); +} + +export async function queryRecentEmailIds( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + afterIso: string, + limit: number, +): Promise { + const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [ + ['Email/query', { + accountId, + filter: { after: afterIso }, + sort: [{ property: 'receivedAt', isAscending: false }], + limit, + calculateTotal: false, + }, 'q'], + ]); + return idsOf(firstResult(responses, 'Email/query')); +} + +// ── calendar ──────────────────────────────────────────────────────────────── + +export async function getCalendarEventsForIndex( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + ids: readonly string[], +): Promise { + if (ids.length === 0) return []; + const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [ + ['CalendarEvent/get', { accountId, ids: [...ids] }, 'g'], + ]); + return listOf(firstResult(responses, 'CalendarEvent/get')); +} + +export async function queryCalendarEventIds( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + afterIso: string, + beforeIso: string, + limit: number, +): Promise { + const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CALENDARS], [ + ['CalendarEvent/query', { + accountId, + // LocalDateTime, per the note in lib/jmap/client.ts:307-312 - Stalwart + // parses these without a zone suffix and ignores unparseable values. + filter: { after: toLocalDateTime(afterIso), before: toLocalDateTime(beforeIso) }, + limit, + calculateTotal: false, + }, 'q'], + ]); + return idsOf(firstResult(responses, 'CalendarEvent/query')); +} + +/** JSCalendar LocalDateTime: `YYYY-MM-DDTHH:MM:SS`, no zone designator. */ +function toLocalDateTime(iso: string): string { + return iso.replace(/\.\d+/, '').replace(/Z$/, '').slice(0, 19); +} + +// ── contacts ──────────────────────────────────────────────────────────────── + +export async function getContactsForIndex( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + ids: readonly string[], +): Promise { + if (ids.length === 0) return []; + const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [ + ['ContactCard/get', { accountId, ids: [...ids] }, 'g'], + ]); + return listOf(firstResult(responses, 'ContactCard/get')); +} + +export async function queryContactIds( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + limit: number, +): Promise { + const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_CONTACTS], [ + ['ContactCard/query', { accountId, limit, calculateTotal: false }, 'q'], + ]); + return idsOf(firstResult(responses, 'ContactCard/query')); +} + +// ── files ─────────────────────────────────────────────────────────────────── + +const FILENODE_INDEX_PROPERTIES = [ + 'id', 'parentId', 'name', 'type', 'blobId', 'size', 'created', 'modified', +] as const; + +export async function getFilesForIndex( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + ids: readonly string[], +): Promise { + if (ids.length === 0) return []; + const responses = await jmapRequest(session, authHeader, [CAP_CORE], [ + ['FileNode/get', { accountId, ids: [...ids], properties: [...FILENODE_INDEX_PROPERTIES] }, 'g'], + ]); + return listOf(firstResult(responses, 'FileNode/get')); +} + +export async function queryFileIds( + session: JmapSessionInfo, + authHeader: string, + accountId: string, + limit: number, +): Promise { + const responses = await jmapRequest(session, authHeader, [CAP_CORE], [ + ['FileNode/query', { accountId, filter: {}, limit, calculateTotal: false }, 'q'], + ]); + return idsOf(firstResult(responses, 'FileNode/query')); +} + +/** + * Builds `id -> "Parent/Child"` paths for the given nodes, walking `parentId` + * upward. FileNode only knows its parent, so the caller has to assemble this; + * unresolvable ancestors just truncate the path rather than failing. + */ +export function buildFilePaths(nodes: readonly FileNode[]): Map { + const byId = new Map(nodes.map((n) => [n.id, n])); + const cache = new Map(); + + const resolve = (id: string, depth: number): string => { + if (depth > 32) return ''; + const cached = cache.get(id); + if (cached !== undefined) return cached; + const node = byId.get(id); + if (!node) return ''; + const parent = node.parentId ? resolve(node.parentId, depth + 1) : ''; + const full = parent ? `${parent}/${node.name}` : node.name; + cache.set(id, full); + return full; + }; + + const out = new Map(); + for (const n of nodes) { + // The document's own `path` metadata is its PARENT directory chain, so a + // search for "Invoices" matches files inside it without the filename + // being duplicated into the body. + out.set(n.id, n.parentId ? resolve(n.parentId, 0) : ''); + } + return out; +} diff --git a/lib/mail-index/key.ts b/lib/mail-index/key.ts new file mode 100644 index 00000000..35c1318b --- /dev/null +++ b/lib/mail-index/key.ts @@ -0,0 +1,174 @@ +// Server-side client for the main process's key service (electron/key-service.ts). +// +// Asks for an account's index key over the inherited fd only when a job needs +// it, and drops it as soon as the job finishes. There is deliberately no cache: +// a resident plaintext key in a long-lived process is exactly the thing the OS +// keychain exists to avoid, and a keychain round trip costs microseconds +// against a job that makes network calls. + +import net from 'node:net'; + +/** Set by electron/main.ts alongside VNCMAIL_DESKTOP_STORE_DIR. */ +export const KEY_FD_ENV = 'VNCMAIL_DESKTOP_KEY_FD'; + +const REQUEST_TIMEOUT_MS = 10_000; + +export type KeyErrorCode = + | 'no-channel' + | 'no-secure-storage' + | 'key-io-failed' + | 'key-unreadable' + | 'bad-request' + | 'timeout'; + +export class IndexKeyError extends Error { + code: KeyErrorCode; + constructor(code: KeyErrorCode, message: string) { + super(message); + this.name = 'IndexKeyError'; + this.code = code; + } +} + +interface Pending { + resolve: (value: { key?: string }) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +} + +let socket: net.Socket | null = null; +let nextId = 1; +const pending = new Map(); +let readBuffer = ''; + +function failAll(error: Error): void { + for (const [, p] of pending) { + clearTimeout(p.timer); + p.reject(error); + } + pending.clear(); +} + +function getSocket(): net.Socket { + if (socket && !socket.destroyed) return socket; + + const raw = process.env[KEY_FD_ENV]?.trim(); + const fd = raw ? Number(raw) : NaN; + if (!Number.isInteger(fd) || fd < 3) { + throw new IndexKeyError( + 'no-channel', + `${KEY_FD_ENV} is not a usable file descriptor (got ${JSON.stringify(raw)}). ` + + `The local index only works inside the Electron desktop shell.`, + ); + } + + let created: net.Socket; + try { + created = new net.Socket({ fd, readable: true, writable: true }); + } catch (error) { + throw new IndexKeyError('no-channel', `Could not open fd ${fd}: ${String(error)}`); + } + // The channel outlives every individual request; don't let it hold the event + // loop open on its own. + created.unref(); + + created.on('data', (chunk: Buffer) => { + readBuffer += chunk.toString('utf8'); + if (readBuffer.length > 64 * 1024) readBuffer = ''; + let newline: number; + while ((newline = readBuffer.indexOf('\n')) >= 0) { + const line = readBuffer.slice(0, newline); + readBuffer = readBuffer.slice(newline + 1); + if (!line.trim()) continue; + let msg: { id?: unknown; ok?: unknown; key?: unknown; code?: unknown; error?: unknown }; + try { + msg = JSON.parse(line); + } catch { + continue; + } + const id = typeof msg.id === 'number' ? msg.id : null; + if (id === null) continue; + const p = pending.get(id); + if (!p) continue; + pending.delete(id); + clearTimeout(p.timer); + if (msg.ok === true) { + p.resolve({ key: typeof msg.key === 'string' ? msg.key : undefined }); + } else { + const code = typeof msg.code === 'string' ? (msg.code as KeyErrorCode) : 'key-io-failed'; + p.reject(new IndexKeyError(code, typeof msg.error === 'string' ? msg.error : 'Key request failed')); + } + } + }); + + const onGone = (error?: Error) => { + socket = null; + readBuffer = ''; + failAll(error ?? new IndexKeyError('no-channel', 'Key service channel closed')); + }; + created.on('close', () => onGone()); + created.on('error', (error) => onGone(new IndexKeyError('no-channel', String(error)))); + + socket = created; + return created; +} + +function request(op: 'getIndexKey' | 'deleteIndexKey', accountId: string): Promise<{ key?: string }> { + const sock = getSocket(); + const id = nextId++; + return new Promise<{ key?: string }>((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(new IndexKeyError('timeout', `Key service did not answer within ${REQUEST_TIMEOUT_MS}ms`)); + }, REQUEST_TIMEOUT_MS); + // Don't let a pending key request keep the process alive either. + timer.unref?.(); + pending.set(id, { resolve, reject, timer }); + try { + sock.write(`${JSON.stringify({ id, op, accountId })}\n`); + } catch (error) { + pending.delete(id); + clearTimeout(timer); + reject(new IndexKeyError('no-channel', `Could not write to the key service: ${String(error)}`)); + } + }); +} + +/** + * Runs `fn` with the account's raw index key, then zeroes the buffer. + * + * Zeroing a Buffer is genuine (unlike a JS string, which cannot be scrubbed) - + * which is why the key crosses the boundary as hex and is converted to a Buffer + * exactly once, here. `store.ts` puts the hex into a `PRAGMA` string, so a copy + * does briefly exist in the JS heap; the buffer wipe bounds how long the + * long-lived copy lives, it does not pretend to eliminate every trace. + */ +export async function withIndexKey( + accountId: string, + fn: (key: Buffer) => Promise | T, +): Promise { + const { key: hex } = await request('getIndexKey', accountId); + if (!hex) throw new IndexKeyError('key-io-failed', 'Key service returned no key'); + const key = Buffer.from(hex, 'hex'); + if (key.length !== 32) { + key.fill(0); + throw new IndexKeyError('key-io-failed', `Key service returned ${key.length} bytes, expected 32`); + } + try { + return await fn(key); + } finally { + key.fill(0); + } +} + +/** Used when purging an account: the key goes FIRST, so an interrupted purge leaves unreadable data. */ +export async function deleteIndexKey(accountId: string): Promise { + await request('deleteIndexKey', accountId); +} + +/** True when this process has a key channel at all (i.e. is the desktop shell's server). */ +export function hasKeyChannel(): boolean { + const raw = process.env[KEY_FD_ENV]?.trim(); + const fd = raw ? Number(raw) : NaN; + return Number.isInteger(fd) && fd >= 3; +} diff --git a/lib/mail-index/paths.ts b/lib/mail-index/paths.ts new file mode 100644 index 00000000..20cc427b --- /dev/null +++ b/lib/mail-index/paths.ts @@ -0,0 +1,51 @@ +// The hosted-deployment gate, and where an account's index file lives. +// +// The standalone Next.js server in `electron/main.ts` is the SAME artifact the +// production `Dockerfile` ships to multi-tenant deployments. An index that +// activated unconditionally would have a shared server start writing every +// user's mail into a server-side SQLite file. So activation is keyed on an env +// var that ONLY `electron/main.ts` sets, and that same var supplies the path - +// one variable doing both jobs, so they cannot drift apart. + +import { createHash } from 'node:crypto'; +import path from 'node:path'; + +/** Set by electron/main.ts on spawn. Absent => the feature does not exist. */ +export const STORE_DIR_ENV = 'VNCMAIL_DESKTOP_STORE_DIR'; + +/** + * The index root, or `null` when this process is not the desktop shell's + * server. Every route must 404 on `null` - not 403, since nothing should learn + * the routes exist in a deployment that doesn't have the feature. + */ +export function getStoreDir(): string | null { + const dir = process.env[STORE_DIR_ENV]?.trim(); + if (!dir) return null; + // Must be absolute: a relative path would resolve against the server's cwd, + // which differs between `electron:dev` and a packaged build. + if (!path.isAbsolute(dir)) return null; + return dir; +} + +/** + * Filenames are a hash, not `username@host`, so a directory listing is not a + * plaintext inventory of the user's accounts. The account id itself lives only + * inside the encrypted file (and in the renderer's own `account-registry`, + * which already stores it in plain localStorage). + */ +export function accountFileToken(accountId: string): string { + return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32); +} + +export function indexDbPath(storeDir: string, accountId: string): string { + return path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`); +} + +export function keyFilePath(storeDir: string, accountId: string): string { + return path.join(storeDir, 'keys', `${accountFileToken(accountId)}.bin`); +} + +/** WAL siblings must be removed with the database, or a purge leaks readable pages. */ +export function dbSiblings(dbPath: string): string[] { + return [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]; +} diff --git a/lib/mail-index/reindex.ts b/lib/mail-index/reindex.ts new file mode 100644 index 00000000..f5a5c4b4 --- /dev/null +++ b/lib/mail-index/reindex.ts @@ -0,0 +1,332 @@ +// The index jobs. +// +// TWO SHAPES, both plain request-scoped work - there is no background worker, +// no cursor, no retry ladder and no resident credential anywhere: +// +// 1. `indexDocuments()` - the PRIMARY path. The renderer's live JMAP push +// connection sees a StateChange, and calls the route with the ids that +// changed (or with no ids, meaning "refetch what's recent for this type"). +// One or a handful of objects, fetched and upserted. +// 2. `catchUpAll()` - the FALLBACK. On app launch, backfill a bounded recent +// window for every supported type, because anything that changed while the +// app was closed produced no push event. +// +// Staleness between refreshes is acceptable by design: this is a search index +// for a retrieval/AI feature, not a mail replica. + +import type { NextRequest } from 'next/server'; +import { generateAccountId } from '@/lib/account-utils'; +import { getStalwartCredentials } from '@/lib/stalwart/credentials'; +import { logger } from '@/lib/logger'; +import { + accountHasCapability, accountIdFor, buildFilePaths, CAP_CALENDARS, CAP_CONTACTS, + CAP_FILENODE, CAP_MAIL, fetchJmapSession, getCalendarEventsForIndex, getContactsForIndex, + getEmailsForIndex, getFilesForIndex, hasCapability, JmapIndexError, queryCalendarEventIds, + queryContactIds, queryFileIds, queryRecentEmailIds, type JmapSessionInfo, +} from './jmap'; +import { extractCalendarEvent, extractContact, extractFile, extractMail } from './extract'; +import { withIndexKey } from './key'; +import { getStoreDir } from './paths'; +import { MailIndex, type ContentType, type IndexDoc } from './store'; + +/** + * Bounded window. Small on purpose: this is the first cut of a retrieval index, + * and a wide window turns "index on every delivery" into a slow request. The + * event-driven path indexes single objects, so the window only bounds catch-up. + */ +export const INDEX_WINDOW_DAYS = 30; +/** Calendar looks forward as well as back - upcoming events are the useful ones. */ +export const CALENDAR_FORWARD_DAYS = 180; +/** Per-type ceiling for one catch-up pass. */ +export const CATCHUP_MAX_PER_TYPE = 500; +/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */ +export const MAX_IDS_PER_CALL = 200; +/** Cap on body bytes requested per message from the server. */ +export const MAX_BODY_VALUE_BYTES = 256_000; +/** Contacts and files have no useful date filter, so they are simply capped. */ +export const CONTACTS_MAX = 2_000; +export const FILES_MAX = 2_000; + +export interface IndexSession { + serverUrl: string; + authHeader: string; + username: string; + slot: number; + /** `username@host` - the durable per-account key. NEVER the cookie slot. */ + accountId: string; +} + +export class IndexSessionError extends Error { + status: number; + constructor(message: string, status: number) { + super(message); + this.name = 'IndexSessionError'; + this.status = status; + } +} + +/** + * Resolves the calling request to an account and a usable Authorization header. + * + * Uses the SAME per-slot encrypted `jmap_stalwart_ctx` cookie that + * `/api/settings`, `/api/push/preview` and `/api/plugin-approval-status` + * already read (`lib/stalwart/credentials.ts`). That cookie is written by + * `/api/auth/stalwart-context`, which the renderer syncs on every login, + * session restore, SSO callback, account switch and token refresh + * (`stores/auth-store.ts`, 10 call sites), and it carries a ready-made header + * for BOTH basic and bearer accounts. + * + * Why this matters beyond convenience: it means the indexer never touches the + * OAuth refresh-token cookie. A server-side refresh would rotate the token into + * a response nobody reads while the browser kept the superseded one, and the + * next real refresh would then fail and log the user out. Reading an + * already-minted header cannot cause that. + */ +export async function resolveIndexSession(request: NextRequest): Promise { + const credentials = await getStalwartCredentials(request); + if (!credentials) { + throw new IndexSessionError('No JMAP auth context for this account; sign in again.', 401); + } + const accountId = generateAccountId(credentials.username, credentials.serverUrl); + return { ...credentials, accountId }; +} + +export interface IndexResult { + accountId: string; + /** Per-type counts of documents written. */ + written: Partial>; + /** Types the server (or this account) doesn't support, so nothing was attempted. */ + skipped: ContentType[]; + /** Non-fatal per-type failures. One broken type must not fail the whole call. */ + errors: Array<{ contentType: ContentType; message: string }>; + durationMs: number; +} + +function isoDaysFromNow(days: number): string { + return new Date(Date.now() + days * 24 * 60 * 60 * 1000).toISOString(); +} + +/** + * Which types this session can actually index. Calendar/contacts are session + * capabilities; files is a PER-ACCOUNT capability (a server can advertise + * filenode while a specific account has it revoked - #563). + */ +export function supportedTypes(session: JmapSessionInfo): { + supported: ContentType[]; + skipped: ContentType[]; + accountIds: Partial>; +} { + const supported: ContentType[] = []; + const skipped: ContentType[] = []; + const accountIds: Partial> = {}; + + const mailAccount = accountIdFor(session, CAP_MAIL); + if (mailAccount) { supported.push('mail'); accountIds.mail = mailAccount; } + else skipped.push('mail'); + + const calAccount = accountIdFor(session, CAP_CALENDARS); + if (calAccount && hasCapability(session, CAP_CALENDARS)) { + supported.push('calendar'); accountIds.calendar = calAccount; + } else skipped.push('calendar'); + + const contactAccount = accountIdFor(session, CAP_CONTACTS); + if (contactAccount && hasCapability(session, CAP_CONTACTS)) { + supported.push('contact'); accountIds.contact = contactAccount; + } else skipped.push('contact'); + + // Files fall back to the mail account id: Stalwart exposes FileNode on the + // same account and does not always list a primaryAccounts entry for it. + const fileAccount = accountIdFor(session, CAP_FILENODE) ?? mailAccount; + if (fileAccount && accountHasCapability(session, fileAccount, CAP_FILENODE)) { + supported.push('file'); accountIds.file = fileAccount; + } else skipped.push('file'); + + return { supported, skipped, accountIds }; +} + +interface FetchArgs { + session: JmapSessionInfo; + authHeader: string; + jmapAccountId: string; + ids: readonly string[] | null; +} + +/** Fetches and flattens one content type. `ids === null` means "the recent window". */ +async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise { + const { session, authHeader, jmapAccountId, ids } = args; + + switch (contentType) { + case 'mail': { + const targetIds = ids ?? await queryRecentEmailIds( + session, authHeader, jmapAccountId, + isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE, + ); + const docs: IndexDoc[] = []; + // Chunked because bodies are big: one Email/get for 500 messages with + // full bodies would be an enormous response. + for (let i = 0; i < targetIds.length; i += 25) { + const emails = await getEmailsForIndex( + session, authHeader, jmapAccountId, targetIds.slice(i, i + 25), MAX_BODY_VALUE_BYTES, + ); + for (const email of emails) docs.push(extractMail(jmapAccountId, email)); + } + return docs; + } + case 'calendar': { + const targetIds = ids ?? await queryCalendarEventIds( + session, authHeader, jmapAccountId, + isoDaysFromNow(-INDEX_WINDOW_DAYS), isoDaysFromNow(CALENDAR_FORWARD_DAYS), + CATCHUP_MAX_PER_TYPE, + ); + const docs: IndexDoc[] = []; + for (let i = 0; i < targetIds.length; i += 50) { + const events = await getCalendarEventsForIndex( + session, authHeader, jmapAccountId, targetIds.slice(i, i + 50), + ); + for (const event of events) docs.push(extractCalendarEvent(jmapAccountId, event)); + } + return docs; + } + case 'contact': { + const targetIds = ids ?? await queryContactIds(session, authHeader, jmapAccountId, CONTACTS_MAX); + const docs: IndexDoc[] = []; + for (let i = 0; i < targetIds.length; i += 100) { + const cards = await getContactsForIndex( + session, authHeader, jmapAccountId, targetIds.slice(i, i + 100), + ); + for (const card of cards) docs.push(extractContact(jmapAccountId, card)); + } + return docs; + } + case 'file': { + const targetIds = ids ?? await queryFileIds(session, authHeader, jmapAccountId, FILES_MAX); + const nodes = []; + for (let i = 0; i < targetIds.length; i += 100) { + nodes.push(...await getFilesForIndex( + session, authHeader, jmapAccountId, targetIds.slice(i, i + 100), + )); + } + // Paths need the whole set in hand, so this one can't stream per chunk. + const paths = buildFilePaths(nodes); + return nodes + // Directories are indexed too: "what's in the Invoices folder" is a + // real query, and a folder row is a few bytes. + .map((node) => extractFile(jmapAccountId, node, { path: paths.get(node.id) })); + } + } +} + +export interface IndexRequest { + /** Types to touch. Empty means every supported type. */ + types?: readonly ContentType[]; + /** + * Per-type ids to index. Omitted/empty for a type means "refetch that type's + * recent window" (the catch-up shape). + */ + ids?: Partial>; + /** Per-type ids to REMOVE (a JMAP `destroyed`). */ + removed?: Partial>; + /** Drop documents outside the retention window after writing. */ + prune?: boolean; +} + +/** + * Runs one index pass. Opens the encrypted store, fetches, upserts, closes. + * + * The key is fetched from the main process for the duration of this call only + * (`withIndexKey`) and zeroed afterwards - there is no cached handle and no + * resident key. + */ +export async function runIndex( + indexSession: IndexSession, + req: IndexRequest, +): Promise { + const started = Date.now(); + const storeDir = getStoreDir(); + if (!storeDir) { + throw new IndexSessionError('The local index is not enabled in this deployment.', 404); + } + + const session = await fetchJmapSession(indexSession.serverUrl, indexSession.authHeader); + + // Identity cross-check. `generateAccountId` used the username from the auth + // context cookie; the server may canonicalise a short login (`linus`) to a + // full address (`linus@example.com`) - which is exactly why AccountEntry + // carries `serverIdentifiers`. Accept either form, reject anything else + // rather than writing one account's mail into another's file. + if (session.username) { + const serverAccountId = generateAccountId(session.username, indexSession.serverUrl); + if (serverAccountId !== indexSession.accountId) { + const shortMatches = session.username.split('@')[0] === indexSession.username.split('@')[0]; + if (!shortMatches) { + throw new IndexSessionError( + 'The JMAP session belongs to a different account than the request cookie.', + 409, + ); + } + } + } + + const { supported, skipped, accountIds } = supportedTypes(session); + const requested = req.types && req.types.length > 0 ? req.types : supported; + const types = requested.filter((t) => supported.includes(t)); + const notAttempted = [...new Set([...skipped, ...requested.filter((t) => !supported.includes(t))])]; + + const written: Partial> = {}; + const errors: IndexResult['errors'] = []; + + await withIndexKey(indexSession.accountId, async (key) => { + const index = MailIndex.open({ storeDir, accountId: indexSession.accountId, key }); + try { + for (const contentType of types) { + const jmapAccountId = accountIds[contentType]; + if (!jmapAccountId) continue; + try { + const removed = req.removed?.[contentType]; + if (removed && removed.length > 0) { + index.remove(jmapAccountId, contentType, removed.slice(0, MAX_IDS_PER_CALL)); + } + + const requestedIds = req.ids?.[contentType]; + const ids = requestedIds && requestedIds.length > 0 + ? requestedIds.slice(0, MAX_IDS_PER_CALL) + : null; + + const docs = await fetchDocs(contentType, { + session, authHeader: indexSession.authHeader, jmapAccountId, ids, + }); + written[contentType] = index.upsert(docs); + + if (req.prune && contentType === 'mail') { + // Only mail prunes by date: calendar's window looks forward, + // contacts have no date, and file rows are metadata-sized. + index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS)); + } + } catch (error) { + // One unsupported or misbehaving type must not fail the others. + const message = error instanceof Error ? error.message : String(error); + errors.push({ contentType, message }); + if (error instanceof JmapIndexError && error.status === 401) throw error; + } + } + } finally { + index.close(); + } + }); + + const result: IndexResult = { + accountId: indexSession.accountId, + written, + skipped: notAttempted, + errors, + durationMs: Date.now() - started, + }; + logger.info('mail-index: pass complete', { + slot: indexSession.slot, + written: JSON.stringify(written), + skipped: notAttempted.join(',') || 'none', + errors: errors.length, + durationMs: result.durationMs, + }); + return result; +} diff --git a/lib/mail-index/store.ts b/lib/mail-index/store.ts new file mode 100644 index 00000000..4e89793d --- /dev/null +++ b/lib/mail-index/store.ts @@ -0,0 +1,444 @@ +// The encrypted local search index: schema, open/close, upsert, search. +// +// One SQLite (SQLCipher) file per account. Rows are ALSO account-scoped +// internally - `(jmap_account_id, content_type, id)` - because a single login +// exposes the user's own JMAP account plus every delegated/shared account, and +// JMAP ids are unique only WITHIN an account (this codebase already works +// around that collision in `lib/jmap/client.ts:388`'s namespaceMailboxIds). +// One file per account keeps purge trivial; the composite key keeps +// delegated accounts from merging inside it. +// +// This is a SEARCH INDEX, not a mail replica. It is allowed to be stale, it is +// allowed to be incomplete, and it can be discarded and rebuilt at any time - +// which is why the schema-version mismatch path below simply drops everything +// rather than migrating. + +import fs from 'node:fs'; +import path from 'node:path'; +import { loadSqlcipher, type SqlcipherDatabase } from './binding'; +import { dbSiblings, indexDbPath } from './paths'; + +export const SCHEMA_VERSION = 1; + +export type ContentType = 'mail' | 'calendar' | 'contact' | 'file'; + +export const CONTENT_TYPES: readonly ContentType[] = ['mail', 'calendar', 'contact', 'file']; + +export function isContentType(v: unknown): v is ContentType { + return typeof v === 'string' && (CONTENT_TYPES as readonly string[]).includes(v); +} + +/** + * One indexable thing, already flattened to text. Produced by the pure + * extractors in `extract.ts` so that every JMAP-shape decision is unit-testable + * without a database or a server. + */ +export interface IndexDoc { + jmapAccountId: string; + contentType: ContentType; + /** JMAP id. Unique only within (jmapAccountId, contentType). */ + id: string; + /** Subject / event title / contact display name / filename. */ + title: string; + /** Addresses and names: sender+recipients, attendees, contact emails/phones, owner. */ + people: string; + /** The bulk searchable text. Plain text only - never HTML. */ + body: string; + /** ISO 8601, or null when the type has no meaningful date. Drives recency ordering. */ + occurredAt: string | null; + /** Small type-specific extras returned verbatim to the caller (never searched). */ + metadata: Record; +} + +export interface SearchHit { + contentType: ContentType; + id: string; + jmapAccountId: string; + title: string; + people: string; + occurredAt: string | null; + metadata: Record; + /** FTS5 bm25 score. Lower is a better match (bm25 returns negative values). */ + score: number; + /** Highlighted excerpt from the body, for feeding an LLM as context. */ + snippet: string; +} + +const DDL = ` +CREATE TABLE IF NOT EXISTS doc ( + jmap_account_id TEXT NOT NULL, + content_type TEXT NOT NULL, + id TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + people TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + occurred_at TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + indexed_at INTEGER NOT NULL, + PRIMARY KEY (jmap_account_id, content_type, id) +); +CREATE INDEX IF NOT EXISTS doc_recent + ON doc(jmap_account_id, content_type, occurred_at DESC); + +-- Standalone (not external-content) FTS5: the text is duplicated into this +-- table and kept in step manually on upsert. External content would avoid the +-- duplication but requires deleting the old FTS row using its OLD column +-- values, which an upsert does not have to hand - a well-known source of +-- silently-stale FTS rows. At this scale (a bounded recent window) the +-- duplication is the cheaper correctness trade. +CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5( + title, people, body, + tokenize='unicode61 remove_diacritics 2' +); + +CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL); +`; + +export class MailIndexUnavailableError extends Error { + constructor(message: string) { + super(message); + this.name = 'MailIndexUnavailableError'; + } +} + +/** + * Assert that the file we just opened is REALLY encrypted. + * + * This is not defensive boilerplate, it guards the sharpest landmine found + * while designing this: on both `node:sqlite` and plain `better-sqlite3`, + * `PRAGMA key = ...` is **silently accepted and does nothing** - no error, a + * working database, and the mail sitting on disk in cleartext. Verified by + * writing a file and recovering a canary string from the raw bytes. + * + * The check is on the VALUE, not the row count: a non-cipher binding returns + * ZERO ROWS for `PRAGMA cipher_version`, so a naive `!== ''` comparison over a + * missing row passes vacuously. Require a non-empty string. + */ +function assertEncrypted(db: SqlcipherDatabase, dbPath: string): void { + const rows = db.pragma('cipher_version'); + const value = + Array.isArray(rows) && rows.length > 0 && rows[0] && typeof rows[0] === 'object' + ? (rows[0] as Record).cipher_version + : undefined; + if (typeof value !== 'string' || value.trim().length === 0) { + db.close(); + throw new MailIndexUnavailableError( + `Refusing to use ${path.basename(dbPath)}: the SQLite binding reports no SQLCipher ` + + `support (PRAGMA cipher_version returned ${JSON.stringify(rows)}), so the index ` + + `would be written in cleartext.`, + ); + } +} + +export interface OpenOptions { + storeDir: string; + accountId: string; + /** Raw 32-byte key. Used as SQLCipher's raw key (no KDF) via `PRAGMA key = "x'..'"`. */ + key: Buffer; +} + +export class MailIndex { + private constructor( + private readonly db: SqlcipherDatabase, + readonly dbPath: string, + ) {} + + /** + * Opens (creating if needed) the account's index. Throws + * MailIndexUnavailableError when the native binding is absent or the file is + * not actually encrypted; the caller turns the feature off rather than + * falling back to something unencrypted. + */ + static open({ storeDir, accountId, key }: OpenOptions): MailIndex { + const Database = loadSqlcipher(); + if (!Database) { + throw new MailIndexUnavailableError( + '@signalapp/sqlcipher is not installed for this platform (it is an optional dependency).', + ); + } + if (key.length !== 32) { + throw new MailIndexUnavailableError(`Index key must be 32 bytes, got ${key.length}.`); + } + + const dbPath = indexDbPath(storeDir, accountId); + fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: 0o700 }); + + let db = new Database(dbPath); + // The key pragma must be the FIRST statement on the connection. Hex form + // means SQLCipher uses these 32 bytes as the raw key with no KDF, which is + // right for a random key (a passphrase would want the KDF). + db.pragma(`key = "x'${key.toString('hex')}'"`); + assertEncrypted(db, dbPath); + + // A wrong key surfaces here rather than at open: SQLCipher only reads the + // header lazily. Treat it as "unreadable" and rebuild from scratch - the + // index is derived data, so there is nothing to recover and never anything + // to prompt the user for (the key was never a user secret). + let version: number | null; + try { + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = NORMAL'); + version = readSchemaVersion(db); + } catch { + db.close(); + for (const f of dbSiblings(dbPath)) { + try { fs.rmSync(f, { force: true }); } catch { /* best effort */ } + } + db = new Database(dbPath); + db.pragma(`key = "x'${key.toString('hex')}'"`); + assertEncrypted(db, dbPath); + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = NORMAL'); + version = null; + } + + if (version !== null && version !== SCHEMA_VERSION) { + // Rebuildable derived data: drop, don't migrate. + db.exec('DROP TABLE IF EXISTS doc_fts; DROP TABLE IF EXISTS doc; DROP TABLE IF EXISTS meta;'); + version = null; + } + if (version === null) { + db.exec(DDL); + db.prepare('INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)').run([ + 'schema_version', + String(SCHEMA_VERSION), + ]); + } + + return new MailIndex(db, dbPath); + } + + close(): void { + try { this.db.close(); } catch { /* already closed */ } + } + + /** + * Upserts documents and keeps the FTS rows in step. Returns the number of + * rows written. One transaction for the whole batch - a partially-applied + * batch is harmless (it is an index) but a transaction is faster. + */ + upsert(docs: readonly IndexDoc[]): number { + if (docs.length === 0) return 0; + + const upsertDoc = this.db.prepare(` + INSERT INTO doc (jmap_account_id, content_type, id, title, people, body, + occurred_at, metadata_json, indexed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(jmap_account_id, content_type, id) DO UPDATE SET + title = excluded.title, people = excluded.people, body = excluded.body, + occurred_at = excluded.occurred_at, metadata_json = excluded.metadata_json, + indexed_at = excluded.indexed_at + RETURNING rowid + `); + const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?'); + const insertFts = this.db.prepare( + 'INSERT INTO doc_fts (rowid, title, people, body) VALUES (?, ?, ?, ?)', + ); + + const now = Date.now(); + let written = 0; + this.db.exec('BEGIN'); + try { + for (const d of docs) { + const row = upsertDoc.get([ + d.jmapAccountId, d.contentType, d.id, + d.title, d.people, d.body, + d.occurredAt, JSON.stringify(d.metadata ?? {}), now, + ]); + const rowid = row?.rowid; + if (typeof rowid !== 'number') continue; + // ON CONFLICT preserves the rowid, so delete-then-insert replaces the + // old FTS row rather than accumulating duplicates for one document. + deleteFts.run([rowid]); + insertFts.run([rowid, d.title, d.people, d.body]); + written++; + } + this.db.exec('COMMIT'); + } catch (error) { + this.db.exec('ROLLBACK'); + throw error; + } + return written; + } + + /** Removes documents by id (a JMAP `destroyed` id, or a stale row). */ + remove(jmapAccountId: string, contentType: ContentType, ids: readonly string[]): number { + if (ids.length === 0) return 0; + const findRow = this.db.prepare( + 'SELECT rowid FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?', + ); + const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?'); + const deleteDoc = this.db.prepare( + 'DELETE FROM doc WHERE jmap_account_id = ? AND content_type = ? AND id = ?', + ); + let removed = 0; + this.db.exec('BEGIN'); + try { + for (const id of ids) { + const row = findRow.get([jmapAccountId, contentType, id]); + if (typeof row?.rowid === 'number') deleteFts.run([row.rowid]); + removed += deleteDoc.run([jmapAccountId, contentType, id]).changes; + } + this.db.exec('COMMIT'); + } catch (error) { + this.db.exec('ROLLBACK'); + throw error; + } + return removed; + } + + /** + * Full-text search - the retrieval surface an AI feature calls to gather + * context. `types` empty/omitted searches everything. + */ + search(opts: { + query: string; + types?: readonly ContentType[]; + limit?: number; + snippetTokens?: number; + }): SearchHit[] { + const match = toFtsMatchQuery(opts.query); + if (!match) return []; + + const limit = Math.min(Math.max(opts.limit ?? 20, 1), 200); + const tokens = Math.min(Math.max(opts.snippetTokens ?? 24, 4), 64); + const types = opts.types && opts.types.length > 0 ? opts.types : null; + const typeFilter = types ? ` AND d.content_type IN (${types.map(() => '?').join(',')})` : ''; + + // bm25 weights: a hit in the title or in a name/address is a stronger + // signal than one in a long body, and for RAG the title is what makes a + // retrieved chunk recognisable. + const rows = this.db + .prepare(` + SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people, + d.occurred_at, d.metadata_json, + bm25(doc_fts, 8.0, 4.0, 1.0) AS score, + snippet(doc_fts, 2, '[', ']', '…', ${tokens}) AS snip + FROM doc_fts + JOIN doc d ON d.rowid = doc_fts.rowid + WHERE doc_fts MATCH ?${typeFilter} + ORDER BY score ASC, d.occurred_at DESC + LIMIT ? + `) + .all([match, ...(types ?? []), limit]); + + return rows.map((r) => ({ + contentType: String(r.content_type) as ContentType, + id: String(r.id), + jmapAccountId: String(r.jmap_account_id), + title: String(r.title ?? ''), + people: String(r.people ?? ''), + occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at), + metadata: safeParseObject(r.metadata_json), + score: typeof r.score === 'number' ? r.score : 0, + snippet: String(r.snip ?? ''), + })); + } + + /** Per-type counts and freshness, for the Settings UI and for debugging. */ + stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> { + return this.db + .prepare(` + SELECT content_type, COUNT(*) AS n, MAX(occurred_at) AS newest, MAX(indexed_at) AS indexed + FROM doc GROUP BY content_type ORDER BY content_type + `) + .all() + .map((r) => ({ + contentType: String(r.content_type), + count: Number(r.n ?? 0), + newest: r.newest === null || r.newest === undefined ? null : String(r.newest), + indexedAt: typeof r.indexed === 'number' ? r.indexed : null, + })); + } + + /** Ids already present, so a catch-up pass can skip re-fetching bodies. */ + existingIds(jmapAccountId: string, contentType: ContentType): Set { + const rows = this.db + .prepare('SELECT id FROM doc WHERE jmap_account_id = ? AND content_type = ?') + .all([jmapAccountId, contentType]); + return new Set(rows.map((r) => String(r.id))); + } + + /** Drops documents older than the retention floor for a type. */ + pruneOlderThan(jmapAccountId: string, contentType: ContentType, isoFloor: string): number { + const rows = this.db + .prepare(` + SELECT rowid FROM doc + WHERE jmap_account_id = ? AND content_type = ? + AND occurred_at IS NOT NULL AND occurred_at < ? + `) + .all([jmapAccountId, contentType, isoFloor]); + if (rows.length === 0) return 0; + const deleteFts = this.db.prepare('DELETE FROM doc_fts WHERE rowid = ?'); + const deleteDoc = this.db.prepare('DELETE FROM doc WHERE rowid = ?'); + this.db.exec('BEGIN'); + try { + for (const r of rows) { + deleteFts.run([r.rowid]); + deleteDoc.run([r.rowid]); + } + this.db.exec('COMMIT'); + } catch (error) { + this.db.exec('ROLLBACK'); + throw error; + } + return rows.length; + } +} + +function readSchemaVersion(db: SqlcipherDatabase): number | null { + try { + const row = db.prepare("SELECT v FROM meta WHERE k = 'schema_version'").get(); + if (!row || row.v === undefined) return null; + const n = Number(row.v); + return Number.isFinite(n) ? n : null; + } catch { + // `meta` doesn't exist yet - a fresh file. + return null; + } +} + +function safeParseObject(v: unknown): Record { + if (typeof v !== 'string') return {}; + try { + const parsed = JSON.parse(v); + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + } catch { + return {}; + } +} + +/** + * Turns arbitrary user text into a safe FTS5 MATCH expression. + * + * FTS5's query syntax is not SQL, so parameter binding does NOT protect it: a + * bare `"` or a stray `*`/`NEAR`/`:` in user input raises + * `fts5: syntax error`, which would turn a normal search box into a 500. Every + * token is quoted (making it a literal phrase) and a trailing `*` is added to + * the last token so typing continues to match as the user types. + * + * Exported for unit testing - it is the one piece of this file with no + * database dependency and the most ways to be wrong. + */ +export function toFtsMatchQuery(raw: string): string | null { + if (typeof raw !== 'string') return null; + // Split on anything that isn't a word character or an intra-word mark. Keeps + // unicode letters (so "Müller" and "東京" survive) via the u flag. + const tokens = raw + .normalize('NFC') + .split(/[^\p{L}\p{N}_@.'-]+/u) + .map((t) => t.replace(/^['-]+|['-]+$/g, '')) + .filter((t) => t.length > 0) + .slice(0, 24); + if (tokens.length === 0) return null; + return tokens + .map((t, i) => { + const quoted = `"${t.replace(/"/g, '""')}"`; + // Prefix-match only the final token, and only if it's long enough to not + // match half the mailbox. + return i === tokens.length - 1 && t.length >= 3 ? `${quoted}*` : quoted; + }) + .join(' AND '); +} diff --git a/next.config.ts b/next.config.ts index bfb6ec08..c578eebf 100644 --- a/next.config.ts +++ b/next.config.ts @@ -50,7 +50,14 @@ const nextConfig: NextConfig = { // esbuild ships native binaries + a README the bundler can't parse; load // it from node_modules at runtime instead of trying to bundle it. Used by // PLUGIN_DEV_DIR's on-the-fly bundler. - serverExternalPackages: ["esbuild"], + // + // @signalapp/sqlcipher is a native N-API addon resolved at runtime by + // node-gyp-build (a directory scan of prebuilds/), which a bundler cannot + // follow. It is also an OPTIONAL dependency - absent on musl/Alpine, where + // both Dockerfiles build - so it must never be a hard build-time import. + // lib/mail-index/binding.ts guards the require; this keeps webpack from + // trying to resolve it at all. + serverExternalPackages: ["esbuild", "@signalapp/sqlcipher"], // Sibling repos checked out under ./repos/ are unrelated source trees that // Turbopack's NFT can otherwise rope into the trace when dynamic fs calls // confuse it. Keeps the build from ballooning memory tracing dead code. diff --git a/package-lock.json b/package-lock.json index 1f784acc..d976188c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -79,6 +79,9 @@ "tw-animate-css": "^1.4.0", "typescript": "^5.9.3", "vitest": "^4.1.5" + }, + "optionalDependencies": { + "@signalapp/sqlcipher": "^4.0.3" } }, "node_modules/@acemir/cssom": { @@ -3372,6 +3375,18 @@ "integrity": "sha512-bXHSaW5jRTmke9Vd0h5P7BtWZG9Znqb8gSDxZnxaGSJnGwPLDPfS+3g0BKzeWqzgZPsIVZkM7m2tbo18cm5HBw==", "license": "MIT" }, + "node_modules/@signalapp/sqlcipher": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@signalapp/sqlcipher/-/sqlcipher-4.0.3.tgz", + "integrity": "sha512-Xp8H+pcOjBacqBh+ohE44gJUJIa/95JqBYWC70A08xhOcqogbnbvweu3gUmyKqNGnVehs7ukeSsuGO6QxdVTVw==", + "hasInstallScript": true, + "license": "AGPL-3.0-only", + "optional": true, + "dependencies": { + "node-addon-api": "*", + "node-gyp-build": "^4.8.4" + } + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -9949,6 +9964,18 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", diff --git a/package.json b/package.json index acea95ea..2d5b0e16 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,9 @@ "webcrypto-liner": "^1.4.3", "zustand": "^5.0.12" }, + "optionalDependencies": { + "@signalapp/sqlcipher": "^4.0.3" + }, "devDependencies": { "@eslint/js": "^9.39.4", "@playwright/test": "^1.59.1", diff --git a/stores/email-store.ts b/stores/email-store.ts index 785a4966..f0b97c0e 100644 --- a/stores/email-store.ts +++ b/stores/email-store.ts @@ -2836,6 +2836,33 @@ export const useEmailStore = create((set, get) => ({ // Update last push update timestamp set({ lastPushUpdate: Date.now() }); + // Feed the desktop shell's encrypted local search index + // (lib/mail-index/**). This is the EVENT-DRIVEN trigger for indexing: the + // push connection is already type-generic (the WS/SSE handlers pass the + // whole `changed` map through, and the WS subscribes with + // dataTypes: null), so mail, calendar, contact and file changes all + // arrive here. Scheduled at the END of this handler, not here, so the + // mail ids it passes come from the ALREADY-REFRESHED list - reading them + // first would hand over the page as it was before the new message + // arrived, i.e. index everything except the delivery that triggered it. + const scheduleIndexUpdate = () => { + void (async () => { + try { + const { indexOnStateChange } = await import('@/lib/mail-index-client'); + const mailIds = get().emails.slice(0, 100).map((e) => e.id); + indexOnStateChange(change, { + // Empty (no mailbox selected yet, or a background account) means + // "no ids to offer" - the server then falls back to its own + // bounded recent-window query rather than indexing nothing. + mailIds: mailIds.length > 0 ? mailIds : undefined, + slot: useAccountStore.getState().getActiveAccount()?.cookieSlot, + }); + } catch { + /* the index is optional; never let it affect mail handling */ + } + })(); + }; + // Get the current account ID from the client (assuming primary account) const accountId = client.getAccountId(); @@ -2896,6 +2923,9 @@ export const useEmailStore = create((set, get) => ({ }); } } + + // Local search index last, with the refreshed ids (see above). + scheduleIndexUpdate(); } catch (error) { console.error('Failed to handle state change:', error); set({