feat(mail-index): encrypted SQLite/FTS5 index over mail, calendar, contacts, files

An on-device, SQLCipher-encrypted full-text index the app can retrieve from to feed an LLM ("prompt against"), for the Electron desktop shell only. Shape: no persistent background worker and no resident credential. Indexing is a normal request-scoped API route, triggered by the renderer's EXISTING live JMAP push connection - so it reacts to each delivery/change rather than polling. - lib/mail-index/binding.ts guarded require of the optional native binding - lib/mail-index/paths.ts the VNCMAIL_DESKTOP_STORE_DIR gate + hashed paths - lib/mail-index/store.ts schema, upsert, FTS5 search, encryption assertion - lib/mail-index/extract.ts PURE JMAP-object -> document extractors - lib/mail-index/jmap.ts minimal stateless server-side JMAP client - lib/mail-index/key.ts per-job key fetch over the inherited fd - lib/mail-index/reindex.ts the job + slot->account resolution - electron/key-service.ts safeStorage wrap/unwrap, served over fd 3 - app/api/offline/reindex POST, event-driven + catch-up - app/api/offline/search GET, the retrieval surface (hits + contextBlock) - lib/mail-index-client.ts renderer client; StateChange -> index call - components/settings/local-index-settings.tsx status + manual catch-up Decisions worth knowing: * `@signalapp/sqlcipher` is an OPTIONAL dependency with a guarded runtime require. It publishes six N-API prebuilds and NO build sources, and both Dockerfiles are node:24-alpine (musl, no matching prebuild) - as a hard dependency it would break the production image and the integration fixture's webmail container, neither of which wants this feature. * Credentials come from the existing per-slot encrypted `jmap_stalwart_ctx` cookie via lib/stalwart/credentials.ts - the same helper /api/settings and /api/push/preview already use. It carries a ready-made header for basic AND bearer accounts, so the indexer never touches the OAuth refresh-token cookie; a server-side refresh would rotate a token into a response nobody reads and silently log the user out. * The encryption key crosses main -> server over an INHERITED FILE DESCRIPTOR, never an environment variable: env is readable by any process running as the same OS user, which would defeat using the OS keychain at all. Fetched per job and zeroed after, so there is no long-lived key copy. * safeStorage's Linux `basic_text` backend (no keyring) is treated as refusal, not degradation - it "encrypts" with a hardcoded public password, which would look like an encrypted mailbox while providing nothing. getSelectedStorageBackend() is Linux-only and platform-guarded. * Every store open asserts `PRAGMA cipher_version` returns a non-empty STRING, not merely a row: a non-cipher binding returns ZERO ROWS, so a row-count check would pass vacuously while writing the mailbox to disk in cleartext. * Files are indexed by name/path/date/size only - NOT by extracted content. Text extraction from arbitrary PDFs/office documents is a separate problem. * Account-scoped composite keys `(jmap_account_id, content_type, id)` are kept even though there is one file per account: one login exposes delegated/shared JMAP accounts too, and JMAP ids are unique only within an account. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-04 23:16:22 +02:00
parent 16466c7296
commit b966d285a9
19 changed files with 2672 additions and 2 deletions
@@ -13,6 +13,7 @@ import { cn } from '@/lib/utils';
 import { getPathPrefix } from '@/lib/browser-navigation';
 import { clearCachedData } from '@/lib/clear-cached-data';
 import { SpamSiegeGame } from './spam-siege-game';
+import { LocalIndexSettings } from './local-index-settings';

 const APP_VERSION = process.env.NEXT_PUBLIC_APP_VERSION || "0.0.0";
 const GIT_COMMIT = process.env.NEXT_PUBLIC_GIT_COMMIT || "unknown";
@@ -218,6 +219,9 @@ export function AboutDataSettings() {
          </Button>
        </SettingItem>
      </SettingsSection>
+
+      {/* Desktop shell only - renders nothing in the browser/PWA build. */}
+      <LocalIndexSettings />
    </>
  );
 }
@@ -0,0 +1,123 @@
+"use client";
+
+// Settings panel for the desktop shell's encrypted local search index.
+//
+// Deliberately small: the index's PRIMARY trigger is the live push connection
+// (see lib/mail-index-client.ts's indexOnStateChange, wired into
+// stores/email-store.ts's handleStateChange), so this panel is a status readout
+// plus a manual catch-up button - not the mechanism.
+//
+// Renders nothing at all outside the Electron shell, where the routes 404.
+
+import { useCallback, useEffect, useState } from 'react';
+import { Button } from '@/components/ui/button';
+import { SettingsSection, SettingItem } from './settings-section';
+import { isElectronShell } from '@/lib/electron-bridge';
+import { useAccountStore } from '@/stores/account-store';
+import { catchUpIndex, fetchIndexStats, type IndexStats } from '@/lib/mail-index-client';
+
+const TYPE_LABELS: Record<string, string> = {
+  mail: 'Mail',
+  calendar: 'Calendar',
+  contact: 'Contacts',
+  file: 'Files',
+};
+
+export function LocalIndexSettings() {
+  const slot = useAccountStore((s) => s.accounts.find((a) => a.id === s.activeAccountId)?.cookieSlot);
+  const [stats, setStats] = useState<IndexStats[] | null>(null);
+  const [busy, setBusy] = useState(false);
+  const [message, setMessage] = useState<string | null>(null);
+  // `null` until the first probe resolves, so we don't flash a panel that then
+  // vanishes on a non-desktop build.
+  const [available, setAvailable] = useState<boolean | null>(null);
+
+  const refreshStats = useCallback(async () => {
+    const next = await fetchIndexStats(slot);
+    setStats(next);
+    setAvailable(next !== null);
+  }, [slot]);
+
+  useEffect(() => {
+    if (!isElectronShell()) {
+      setAvailable(false);
+      return;
+    }
+    void refreshStats();
+  }, [refreshStats]);
+
+  const handleRebuild = async () => {
+    setBusy(true);
+    setMessage(null);
+    try {
+      const result = await catchUpIndex(slot);
+      if (result.unavailable) {
+        setAvailable(false);
+        setMessage(result.error ?? 'The encrypted index is unavailable on this system.');
+        return;
+      }
+      if (!result.ok) {
+        setMessage(result.error ?? 'Indexing failed.');
+        return;
+      }
+      const written = Object.entries(result.written ?? {})
+        .map(([type, n]) => `${TYPE_LABELS[type] ?? type}: ${n}`)
+        .join(', ');
+      const failed = (result.errors ?? []).map((e) => `${e.contentType} (${e.message})`).join('; ');
+      setMessage(
+        [
+          written ? `Indexed ${written}.` : 'Nothing to index.',
+          result.skipped?.length ? `Not supported: ${result.skipped.join(', ')}.` : '',
+          failed ? `Problems: ${failed}` : '',
+        ]
+          .filter(Boolean)
+          .join(' '),
+      );
+      await refreshStats();
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  if (available === false || available === null) return null;
+
+  const total = (stats ?? []).reduce((sum, s) => sum + s.count, 0);
+
+  return (
+    <SettingsSection
+      title="Local search index"
+      description={
+        'An encrypted index of your recent mail, calendar events, contacts and file names, ' +
+        'stored on this device only. It updates automatically as items arrive, and powers ' +
+        'local search and AI answers about your own data. Files are indexed by name and ' +
+        'location, not by their contents.'
+      }
+    >
+      <SettingItem
+        label="Indexed items"
+        description={
+          total > 0
+            ? (stats ?? [])
+                .map((s) => `${TYPE_LABELS[s.contentType] ?? s.contentType}: ${s.count}`)
+                .join(' · ')
+            : 'Nothing indexed yet.'
+        }
+      >
+        <span className="text-sm text-muted-foreground tabular-nums">{total}</span>
+      </SettingItem>
+
+      <SettingItem
+        label="Update now"
+        description={
+          message ??
+          'Catches up on anything that changed while the app was closed. Normally not needed - ' +
+            'the index updates itself when mail, events, contacts or files change.'
+        }
+      >
+        <Button variant="outline" size="sm" onClick={handleRebuild} disabled={busy}>
+          {busy ? 'Indexing…' : 'Update index'}
+        </Button>
+      </SettingItem>
+    </SettingsSection>
+  );
+}