promote: retention/recency fixes, supervised OpenCode + provider management (dev→main)
This commit is contained in:
@@ -1088,9 +1088,10 @@ export default function Home() {
|
||||
const runCatchUp = async (attempt: number) => {
|
||||
if (catchUpCancelled) return;
|
||||
try {
|
||||
const { catchUpIndex } = await import('@/lib/mail-index-client');
|
||||
const { catchUpIndex, getRetentionDays } = await import('@/lib/mail-index-client');
|
||||
const result = await catchUpIndex(
|
||||
useAccountStore.getState().getActiveAccount()?.cookieSlot,
|
||||
getRetentionDays(),
|
||||
);
|
||||
if (!result.ok && !result.unavailable && attempt + 1 < catchUpRetryDelaysMs.length) {
|
||||
catchUpTimer = setTimeout(() => void runCatchUp(attempt + 1), catchUpRetryDelaysMs[attempt + 1]);
|
||||
|
||||
@@ -54,7 +54,7 @@ export async function POST(request: NextRequest) {
|
||||
const found = await findOpencodeServer();
|
||||
if (!found) {
|
||||
return NextResponse.json(
|
||||
{ error: 'No local OpenCode server found. Start one with: opencode serve --port 4096' },
|
||||
{ error: 'No local OpenCode server is running. The desktop app starts one automatically when the opencode CLI is installed \u2014 install it from opencode.ai, then restart VNCmail+.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ export async function GET(request: NextRequest) {
|
||||
// isn't running), and the client turns it into setup guidance rather than
|
||||
// an error banner.
|
||||
return NextResponse.json(
|
||||
{ error: 'No local OpenCode server found. Start one with: opencode serve --port 4096' },
|
||||
{ error: 'No local OpenCode server is running. The desktop app starts one automatically when the opencode CLI is installed \u2014 install it from opencode.ai, then restart VNCmail+.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { getStalwartCredentials } from '@/lib/stalwart/credentials';
|
||||
import { configManager } from '@/lib/admin/config-manager';
|
||||
import {
|
||||
findOpencodeServer, listOpencodeProviders, setOpencodeProviderKey, removeOpencodeProvider,
|
||||
} from '@/lib/ai/opencode';
|
||||
import { logger } from '@/lib/logger';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
const SETUP_ERROR =
|
||||
'No local OpenCode server is running. The desktop app starts one automatically when the opencode CLI is installed — install it from opencode.ai, then restart VNCmail+.';
|
||||
|
||||
async function requireOpencode(request: NextRequest) {
|
||||
const auth = await getStalwartCredentials(request);
|
||||
if (!auth) return { error: NextResponse.json({ error: 'not authenticated' }, { status: 401 }) } as const;
|
||||
|
||||
await configManager.ensureLoaded();
|
||||
if (configManager.getAiConsoleConfig().classesEnabled.opencode === false) {
|
||||
return { error: NextResponse.json({ error: 'the OpenCode class is disabled by admin policy' }, { status: 403 }) } as const;
|
||||
}
|
||||
|
||||
const found = await findOpencodeServer();
|
||||
if (!found) return { error: NextResponse.json({ error: SETUP_ERROR }, { status: 503 }) } as const;
|
||||
return { baseUrl: found.baseUrl } as const;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET/PUT/DELETE /api/ai/opencode/providers — lets a user add "any LLM
|
||||
* OpenCode supports" from inside this app, rather than only whatever was
|
||||
* already authenticated via its own CLI. See lib/ai/opencode.ts's module
|
||||
* note on why this only covers API-key providers for now, not OAuth ones.
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
const result = await requireOpencode(request);
|
||||
if ('error' in result) return result.error;
|
||||
try {
|
||||
const providers = await listOpencodeProviders(result.baseUrl);
|
||||
return NextResponse.json({ providers }, { headers: { 'Cache-Control': 'no-store' } });
|
||||
} catch (cause) {
|
||||
logger.error('opencode providers list failed', { error: cause instanceof Error ? cause.message : String(cause) });
|
||||
return NextResponse.json({ error: 'Could not list OpenCode providers' }, { status: 502 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest) {
|
||||
const result = await requireOpencode(request);
|
||||
if ('error' in result) return result.error;
|
||||
|
||||
let body: { providerID?: unknown; key?: unknown };
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'invalid JSON body' }, { status: 400 });
|
||||
}
|
||||
const providerID = typeof body.providerID === 'string' ? body.providerID.trim() : '';
|
||||
const key = typeof body.key === 'string' ? body.key.trim() : '';
|
||||
if (!providerID || !key) {
|
||||
return NextResponse.json({ error: 'providerID and key are required' }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
await setOpencodeProviderKey(result.baseUrl, providerID, key);
|
||||
// VERIFY rather than trust the 200: OpenCode accepts a bare API key for
|
||||
// every provider (confirmed live), but does not consider every provider
|
||||
// "connected" from that alone - Snowflake Cortex, for one real example,
|
||||
// needs SNOWFLAKE_ACCOUNT alongside its token, and a single key field
|
||||
// silently leaves it unconnected with no error from the PUT itself. The
|
||||
// provider's own `env` array length does NOT predict this reliably either
|
||||
// (Azure needs two env vars and DOES connect from one key) - the only
|
||||
// honest source of truth is asking OpenCode again.
|
||||
const after = await listOpencodeProviders(result.baseUrl);
|
||||
const nowConnected = after.find((p) => p.id === providerID)?.connected === true;
|
||||
if (!nowConnected) {
|
||||
return NextResponse.json({
|
||||
ok: false,
|
||||
error: `OpenCode stored the key but does not show ${providerID} as connected — it likely needs more than one credential field (check its requirements with the opencode CLI: opencode auth login ${providerID}).`,
|
||||
}, { status: 200 });
|
||||
}
|
||||
return NextResponse.json({ ok: true });
|
||||
} catch (cause) {
|
||||
logger.error('opencode provider auth failed', { providerID, error: cause instanceof Error ? cause.message : String(cause) });
|
||||
return NextResponse.json({ error: cause instanceof Error ? cause.message : 'Could not add the provider' }, { status: 502 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function DELETE(request: NextRequest) {
|
||||
const result = await requireOpencode(request);
|
||||
if ('error' in result) return result.error;
|
||||
|
||||
const providerID = request.nextUrl.searchParams.get('providerID')?.trim();
|
||||
if (!providerID) {
|
||||
return NextResponse.json({ error: 'providerID is required' }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
await removeOpencodeProvider(result.baseUrl, providerID);
|
||||
return NextResponse.json({ ok: true });
|
||||
} catch (cause) {
|
||||
logger.error('opencode provider removal failed', { providerID, error: cause instanceof Error ? cause.message : String(cause) });
|
||||
return NextResponse.json({ error: cause instanceof Error ? cause.message : 'Could not remove the provider' }, { status: 502 });
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,7 @@ import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
||||
import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key';
|
||||
import { getStoreDir } from '@/lib/mail-index/paths';
|
||||
import {
|
||||
IndexSessionError, MAX_IDS_PER_CALL, resolveIndexSession, runIndex,
|
||||
IndexSessionError, MAX_IDS_PER_CALL, normalizeWindowDays, resolveIndexSession, runIndex,
|
||||
type IndexRequest,
|
||||
} from '@/lib/mail-index/reindex';
|
||||
import { CONTENT_TYPES, isContentType, type ContentType } from '@/lib/mail-index/store';
|
||||
@@ -73,6 +73,10 @@ export async function POST(request: NextRequest) {
|
||||
removed: parseIdMap(body.removed),
|
||||
// Pruning is a catch-up concern; a single-delivery call shouldn't scan.
|
||||
prune: body.catchUp === true,
|
||||
// `undefined` (absent) means "use the default"; an explicit null means
|
||||
// keep everything. normalizeWindowDays() in runIndex clamps anything
|
||||
// unexpected, since this value drives deletion.
|
||||
windowDays: body.windowDays === undefined ? undefined : normalizeWindowDays(body.windowDays),
|
||||
};
|
||||
|
||||
try {
|
||||
|
||||
@@ -17,6 +17,7 @@ import { IndexSessionError, resolveIndexSession } from '@/lib/mail-index/reindex
|
||||
import {
|
||||
isContentType, MailIndex, MailIndexUnavailableError, type ContentType, type SearchHit,
|
||||
} from '@/lib/mail-index/store';
|
||||
import { detectRecencyIntent } from '@/lib/mail-index/recency';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
@@ -84,7 +85,29 @@ export async function GET(request: NextRequest) {
|
||||
// not deliberate search-box keywords, so strict AND-every-token
|
||||
// matching (the default) drops nearly all of them. See
|
||||
// toFtsMatchQueryAny's docstring for the confirmed-live failure.
|
||||
return { hits: index.search({ query, types, limit, mode: 'any' }), stats: wantStats ? stats : undefined };
|
||||
const keywordHits = index.search({ query, types, limit, mode: 'any' });
|
||||
|
||||
// RECENCY leg. Keyword search structurally cannot answer "the last
|
||||
// mail" or "everything from July" (see lib/mail-index/recency.ts), so
|
||||
// when the question is really about time, add a date-ordered slice.
|
||||
// ADDED to the keyword hits rather than replacing them: "what did the
|
||||
// last mail from Anna say" is both a time question and a content one.
|
||||
const intent = detectRecencyIntent(query);
|
||||
if (!intent) {
|
||||
return { hits: keywordHits, stats: wantStats ? stats : undefined };
|
||||
}
|
||||
const recentHits = index.recent({
|
||||
types, limit: Math.min(intent.limit, limit * 3), since: intent.since, until: intent.until,
|
||||
});
|
||||
const seen = new Set(keywordHits.map((h) => `${h.contentType}:${h.id}`));
|
||||
const merged = [...keywordHits];
|
||||
for (const hit of recentHits) {
|
||||
const key = `${hit.contentType}:${hit.id}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
merged.push(hit);
|
||||
}
|
||||
return { hits: merged, stats: wantStats ? stats : undefined, recency: intent };
|
||||
} finally {
|
||||
index.close();
|
||||
}
|
||||
@@ -100,6 +123,10 @@ export async function GET(request: NextRequest) {
|
||||
// Everything a prompt needs, pre-joined in rank order.
|
||||
contextBlock: payload.hits.map(toContextBlock).join('\n\n---\n\n'),
|
||||
...(payload.stats ? { stats: payload.stats } : {}),
|
||||
// Present when the question was read as a time question — lets the
|
||||
// client say "these are the newest N" instead of implying relevance
|
||||
// ranking it did not do.
|
||||
...(payload.recency ? { recency: payload.recency } : {}),
|
||||
},
|
||||
{ headers: { 'Cache-Control': 'no-store' } },
|
||||
);
|
||||
|
||||
@@ -23,6 +23,10 @@ import {
|
||||
listLocalModels,
|
||||
listServerModels,
|
||||
listOpencodeModels,
|
||||
listOpencodeProviders,
|
||||
addOpencodeProvider,
|
||||
removeOpencodeProvider,
|
||||
type OpencodeProviderOption,
|
||||
type OpencodeModelOption,
|
||||
testLocalConnection,
|
||||
type AskResult,
|
||||
@@ -154,6 +158,58 @@ export function AiAssistantSettings() {
|
||||
const [refreshingOpencode, setRefreshingOpencode] = useState(false);
|
||||
const [opencodeError, setOpencodeError] = useState<string | null>(null);
|
||||
|
||||
// ── OpenCode provider management — "add any LLM OpenCode supports" from
|
||||
// inside this app, not only whatever its own CLI already authenticated. ──
|
||||
const [opencodeProviders, setOpencodeProviders] = useState<OpencodeProviderOption[]>([]);
|
||||
const [loadingProviders, setLoadingProviders] = useState(false);
|
||||
const [providerSearch, setProviderSearch] = useState('');
|
||||
const [addingProviderId, setAddingProviderId] = useState<string | null>(null);
|
||||
const [newProviderKey, setNewProviderKey] = useState('');
|
||||
const [providerBusyId, setProviderBusyId] = useState<string | null>(null);
|
||||
const [providerActionError, setProviderActionError] = useState<string | null>(null);
|
||||
const [showProviderManager, setShowProviderManager] = useState(false);
|
||||
|
||||
const refreshOpencodeProviders = useCallback(async () => {
|
||||
setLoadingProviders(true);
|
||||
setProviderActionError(null);
|
||||
try {
|
||||
setOpencodeProviders(await listOpencodeProviders());
|
||||
} catch (err) {
|
||||
setProviderActionError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setLoadingProviders(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleAddProvider = useCallback(async (providerId: string) => {
|
||||
if (!newProviderKey.trim()) return;
|
||||
setProviderBusyId(providerId);
|
||||
setProviderActionError(null);
|
||||
try {
|
||||
await addOpencodeProvider(providerId, newProviderKey.trim());
|
||||
setAddingProviderId(null);
|
||||
setNewProviderKey('');
|
||||
await refreshOpencodeProviders();
|
||||
} catch (err) {
|
||||
setProviderActionError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setProviderBusyId(null);
|
||||
}
|
||||
}, [newProviderKey, refreshOpencodeProviders]);
|
||||
|
||||
const handleRemoveProvider = useCallback(async (providerId: string) => {
|
||||
setProviderBusyId(providerId);
|
||||
setProviderActionError(null);
|
||||
try {
|
||||
await removeOpencodeProvider(providerId);
|
||||
await refreshOpencodeProviders();
|
||||
} catch (err) {
|
||||
setProviderActionError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setProviderBusyId(null);
|
||||
}
|
||||
}, [refreshOpencodeProviders]);
|
||||
|
||||
const refreshOpencodeModels = useCallback(async () => {
|
||||
setRefreshingOpencode(true);
|
||||
setOpencodeError(null);
|
||||
@@ -423,6 +479,109 @@ export function AiAssistantSettings() {
|
||||
</span>
|
||||
</SettingItem>
|
||||
)}
|
||||
|
||||
<SettingItem
|
||||
label="Providers"
|
||||
description="Add credentials for any provider OpenCode supports — a key entered here is stored by OpenCode itself, not by this app. Providers that only offer a browser sign-in (OAuth) aren't manageable here yet; use the opencode CLI for those."
|
||||
>
|
||||
<Button
|
||||
variant="outline" size="sm"
|
||||
onClick={() => {
|
||||
const next = !showProviderManager;
|
||||
setShowProviderManager(next);
|
||||
if (next && opencodeProviders.length === 0) void refreshOpencodeProviders();
|
||||
}}
|
||||
>
|
||||
{showProviderManager ? 'Hide' : 'Manage providers'}
|
||||
</Button>
|
||||
</SettingItem>
|
||||
|
||||
{showProviderManager && (
|
||||
<div className="px-4 pb-4 space-y-3">
|
||||
{providerActionError && (
|
||||
<p className="flex items-start gap-1.5 text-sm text-destructive">
|
||||
<AlertTriangle className="w-3.5 h-3.5 shrink-0 mt-0.5" /> {providerActionError}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={providerSearch}
|
||||
onChange={(e) => setProviderSearch(e.target.value)}
|
||||
placeholder="Search providers (e.g. anthropic, openai, groq)…"
|
||||
spellCheck={false}
|
||||
className={inputClass}
|
||||
/>
|
||||
<Button variant="outline" size="sm" onClick={refreshOpencodeProviders} disabled={loadingProviders}>
|
||||
<RefreshCw className={`w-3.5 h-3.5 me-1.5 ${loadingProviders ? 'animate-spin' : ''}`} />
|
||||
Refresh
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{opencodeProviders.length === 0 && !loadingProviders && (
|
||||
<p className="text-xs text-muted-foreground">No providers loaded yet — click Refresh.</p>
|
||||
)}
|
||||
|
||||
<div className="max-h-72 overflow-y-auto space-y-1.5">
|
||||
{opencodeProviders
|
||||
.filter((p) => {
|
||||
const q = providerSearch.trim().toLowerCase();
|
||||
return !q || p.id.toLowerCase().includes(q) || p.name.toLowerCase().includes(q);
|
||||
})
|
||||
// Connected first (already sorted server-side), then cap what
|
||||
// renders — 180 providers in one scroll box is noise, not choice.
|
||||
.slice(0, providerSearch.trim() ? 40 : 20)
|
||||
.map((p) => (
|
||||
<div key={p.id} className="flex items-center gap-2 rounded-md border border-border px-3 py-2">
|
||||
<div className="flex-1 min-w-0">
|
||||
<span className="text-sm">{p.name}</span>
|
||||
<span className="ms-1.5 text-xs text-muted-foreground">{p.id}</span>
|
||||
</div>
|
||||
{p.connected ? (
|
||||
<>
|
||||
<span className="flex items-center gap-1 text-xs text-emerald-600 dark:text-emerald-500">
|
||||
<CheckCircle className="w-3.5 h-3.5" /> Connected
|
||||
</span>
|
||||
<Button
|
||||
variant="outline" size="sm"
|
||||
onClick={() => handleRemoveProvider(p.id)}
|
||||
disabled={providerBusyId === p.id}
|
||||
>
|
||||
<Trash2 className="w-3.5 h-3.5" />
|
||||
</Button>
|
||||
</>
|
||||
) : p.supportsApiKey ? (
|
||||
addingProviderId === p.id ? (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<input
|
||||
type="password"
|
||||
value={newProviderKey}
|
||||
onChange={(e) => setNewProviderKey(e.target.value)}
|
||||
placeholder="API key"
|
||||
autoFocus
|
||||
className="px-2 py-1 text-xs rounded-md bg-muted border border-border w-36"
|
||||
/>
|
||||
<Button size="sm" onClick={() => handleAddProvider(p.id)} disabled={providerBusyId === p.id || !newProviderKey.trim()}>
|
||||
Save
|
||||
</Button>
|
||||
<Button variant="outline" size="sm" onClick={() => { setAddingProviderId(null); setNewProviderKey(''); }}>
|
||||
<X className="w-3.5 h-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
) : (
|
||||
<Button variant="outline" size="sm" onClick={() => { setAddingProviderId(p.id); setNewProviderKey(''); }}>
|
||||
<Plus className="w-3.5 h-3.5 me-1" /> Add key
|
||||
</Button>
|
||||
)
|
||||
) : (
|
||||
<span className="text-xs text-muted-foreground">Browser sign-in only</span>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</SettingsSection>
|
||||
)}
|
||||
|
||||
|
||||
@@ -14,7 +14,9 @@ import { Button } from '@/components/ui/button';
|
||||
import { SettingsSection, SettingItem } from './settings-section';
|
||||
import { isElectronShell } from '@/lib/electron-bridge';
|
||||
import { useAccountStore } from '@/stores/account-store';
|
||||
import { catchUpIndex, fetchIndexStats, type IndexStats } from '@/lib/mail-index-client';
|
||||
import {
|
||||
catchUpIndex, fetchIndexStats, getRetentionDays, setRetentionDays, type IndexStats,
|
||||
} from '@/lib/mail-index-client';
|
||||
import {
|
||||
chainSync, fetchReplicaStatus, purgeReplica, updateRetentionPolicy,
|
||||
type ReplicaStatus, type RetentionPolicy,
|
||||
@@ -35,6 +37,10 @@ export function LocalIndexSettings() {
|
||||
// `null` until the first probe resolves, so we don't flash a panel that then
|
||||
// vanishes on a non-desktop build.
|
||||
const [available, setAvailable] = useState<boolean | null>(null);
|
||||
// `null` = keep everything. Read once on mount; the setter writes through.
|
||||
const [retentionDays, setRetentionDaysState] = useState<number | null>(365);
|
||||
|
||||
useEffect(() => { setRetentionDaysState(getRetentionDays()); }, []);
|
||||
|
||||
const refreshStats = useCallback(async () => {
|
||||
const next = await fetchIndexStats(slot);
|
||||
@@ -54,7 +60,7 @@ export function LocalIndexSettings() {
|
||||
setBusy(true);
|
||||
setMessage(null);
|
||||
try {
|
||||
const result = await catchUpIndex(slot);
|
||||
const result = await catchUpIndex(slot, retentionDays);
|
||||
if (result.unavailable) {
|
||||
setAvailable(false);
|
||||
setMessage(result.error ?? 'The encrypted index is unavailable on this system.');
|
||||
@@ -110,6 +116,32 @@ export function LocalIndexSettings() {
|
||||
<span className="text-sm text-muted-foreground tabular-nums">{total}</span>
|
||||
</SettingItem>
|
||||
|
||||
<SettingItem
|
||||
label="Keep AI search history for"
|
||||
description={
|
||||
'How far back the AI assistant can search your mail. This also PRUNES: mail older ' +
|
||||
'than the window is removed from the local index on the next update, so a short ' +
|
||||
'window means questions about older mail cannot be answered. Only ever covers the ' +
|
||||
'mailbox you are signed in to.'
|
||||
}
|
||||
>
|
||||
<select
|
||||
className="h-9 rounded-md border border-border bg-background px-2 text-sm"
|
||||
value={retentionDays === null ? 'forever' : String(retentionDays)}
|
||||
onChange={(e) => {
|
||||
const next = e.target.value === 'forever' ? null : Number.parseInt(e.target.value, 10);
|
||||
setRetentionDaysState(next);
|
||||
setRetentionDays(next);
|
||||
setMessage('Saved. Choose "Update index" to apply it now.');
|
||||
}}
|
||||
>
|
||||
<option value="30">30 days</option>
|
||||
<option value="90">3 months</option>
|
||||
<option value="365">1 year</option>
|
||||
<option value="forever">Everything</option>
|
||||
</select>
|
||||
</SettingItem>
|
||||
|
||||
<SettingItem
|
||||
label="Update now"
|
||||
description={
|
||||
|
||||
@@ -5,4 +5,4 @@ kind: Component
|
||||
images:
|
||||
- name: vncmail-plus
|
||||
newName: registry.gitlab.vnc.biz/gitlab-instance-b9b5cf2f/vncmail-plus
|
||||
newTag: sha-1f199fdc
|
||||
newTag: sha-35ed6a28
|
||||
|
||||
@@ -160,6 +160,89 @@ function ensureSessionSecretFile(): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
// ── OpenCode agent server ────────────────────────────────────────────────
|
||||
//
|
||||
// The `opencode` AI class talks to a locally-running `opencode serve`. Left to
|
||||
// the user that means "remember to start a terminal process before opening
|
||||
// your mail app, and again after every reboot" - which is to say the feature
|
||||
// quietly stops existing. So the desktop shell owns its lifecycle: start it if
|
||||
// the binary is installed, restart it if it dies, kill it on quit.
|
||||
//
|
||||
// SECURITY: opencode itself warns "OPENCODE_SERVER_PASSWORD is not set; server
|
||||
// is unsecured" - without one, any local process can drive the agent. We always
|
||||
// generate one. Auth is HTTP Basic with the username EXACTLY `opencode`
|
||||
// (verified against 1.18.14: an empty or arbitrary username 401s even with the
|
||||
// right password, and no bearer/custom-header form works) - undocumented in its
|
||||
// own OpenAPI spec, which declares no securitySchemes at all.
|
||||
|
||||
let opencodeProcess: ChildProcess | null = null;
|
||||
let opencodeRestarts = 0;
|
||||
/** Set by stopOpencodeServer() so the exit handler can tell a deliberate
|
||||
* shutdown from a crash and not fight the quit by respawning. */
|
||||
let opencodeStopping = false;
|
||||
const OPENCODE_MAX_RESTARTS = 3;
|
||||
|
||||
/** Where the binary lives. `~/.opencode/bin` is its own installer's default and
|
||||
* is NOT on the PATH a GUI app inherits on macOS, so PATH alone finds nothing
|
||||
* for most users. */
|
||||
function findOpencodeBinary(): string | null {
|
||||
const explicit = process.env.OPENCODE_BIN?.trim();
|
||||
if (explicit && fs.existsSync(explicit)) return explicit;
|
||||
const candidates = [
|
||||
path.join(app.getPath("home"), ".opencode", "bin", "opencode"),
|
||||
"/opt/homebrew/bin/opencode",
|
||||
"/usr/local/bin/opencode",
|
||||
"/usr/bin/opencode",
|
||||
];
|
||||
return candidates.find((c) => fs.existsSync(c)) ?? null;
|
||||
}
|
||||
|
||||
interface OpencodeHandle {
|
||||
baseUrl: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
async function startOpencodeServer(): Promise<OpencodeHandle | null> {
|
||||
const binary = findOpencodeBinary();
|
||||
if (!binary) return null; // not installed - the class simply stays unavailable
|
||||
|
||||
const port = await getFreePort();
|
||||
// Per-launch, never persisted: the server dies with the app, so there is no
|
||||
// value in a durable secret and every reason not to leave one on disk.
|
||||
const password = randomBytes(24).toString("hex");
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
|
||||
const spawnOnce = () => {
|
||||
opencodeProcess = spawn(binary, ["serve", "--port", String(port), "--hostname", "127.0.0.1"], {
|
||||
env: { ...process.env, OPENCODE_SERVER_PASSWORD: password },
|
||||
stdio: "ignore",
|
||||
});
|
||||
opencodeProcess.on("exit", (code, signal) => {
|
||||
opencodeProcess = null;
|
||||
// A deliberate shutdown arrives as SIGTERM from stopOpencodeServer().
|
||||
if (opencodeStopping || signal === "SIGTERM") return;
|
||||
if (opencodeRestarts >= OPENCODE_MAX_RESTARTS) {
|
||||
console.error(`[opencode] gave up restarting after ${OPENCODE_MAX_RESTARTS} attempts (last code=${code})`);
|
||||
return;
|
||||
}
|
||||
opencodeRestarts += 1;
|
||||
console.error(`[opencode] server exited (code=${code}); restart ${opencodeRestarts}/${OPENCODE_MAX_RESTARTS}`);
|
||||
setTimeout(spawnOnce, 1000 * opencodeRestarts);
|
||||
});
|
||||
};
|
||||
spawnOnce();
|
||||
|
||||
return { baseUrl, password };
|
||||
}
|
||||
|
||||
function stopOpencodeServer(): void {
|
||||
opencodeStopping = true;
|
||||
if (!opencodeProcess) return;
|
||||
const proc = opencodeProcess;
|
||||
opencodeProcess = null;
|
||||
proc.kill("SIGTERM");
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates the standalone server's entrypoint. Packaged builds ship it as an
|
||||
* extraResource (see electron-builder.config.js) because .next/standalone
|
||||
@@ -256,6 +339,9 @@ async function startStandaloneServer(): Promise<string> {
|
||||
// using the OS keychain at all. The fd NUMBER below is not a secret; only
|
||||
// what travels over it is.
|
||||
const sessionSecretFile = ensureSessionSecretFile();
|
||||
// Started before the app server so its address can be handed over as env;
|
||||
// null when opencode isn't installed, in which case the class stays absent.
|
||||
const opencode = await startOpencodeServer();
|
||||
|
||||
serverProcess = spawn(process.execPath, [serverEntry], {
|
||||
env: {
|
||||
@@ -268,6 +354,9 @@ async function startStandaloneServer(): Promise<string> {
|
||||
// SESSION_SECRET env var outranks any file in getSessionSecret()'s
|
||||
// resolution order regardless).
|
||||
...(sessionSecretFile ? { SESSION_SECRET_FILE: sessionSecretFile } : {}),
|
||||
...(opencode
|
||||
? { OPENCODE_BASE_URL: opencode.baseUrl, OPENCODE_SERVER_PASSWORD: opencode.password }
|
||||
: {}),
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: "1",
|
||||
PORT: String(port),
|
||||
@@ -408,6 +497,7 @@ app.whenReady().then(() => {
|
||||
|
||||
app.on("window-all-closed", () => {
|
||||
stopStandaloneServer();
|
||||
stopOpencodeServer();
|
||||
if (process.platform !== "darwin") {
|
||||
app.quit();
|
||||
}
|
||||
@@ -415,6 +505,7 @@ app.on("window-all-closed", () => {
|
||||
|
||||
app.on("before-quit", () => {
|
||||
stopStandaloneServer();
|
||||
stopOpencodeServer();
|
||||
});
|
||||
|
||||
app.on("activate", () => {
|
||||
|
||||
@@ -68,6 +68,41 @@ describe('opencodeBaseUrls', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('auth', () => {
|
||||
const originalFetch = global.fetch;
|
||||
const originalPw = process.env.OPENCODE_SERVER_PASSWORD;
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
if (originalPw === undefined) delete process.env.OPENCODE_SERVER_PASSWORD;
|
||||
else process.env.OPENCODE_SERVER_PASSWORD = originalPw;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('sends HTTP Basic with the username EXACTLY "opencode"', async () => {
|
||||
// Verified against 1.18.14: an empty or arbitrary username 401s even with
|
||||
// the right password, and no bearer/custom-header form works. Its OpenAPI
|
||||
// spec declares no securitySchemes, so this is only knowable by trying it
|
||||
// - which makes it exactly the kind of thing to pin with a test.
|
||||
process.env.OPENCODE_SERVER_PASSWORD = 'hunter2';
|
||||
const fetchMock = vi.fn().mockResolvedValue(jsonResponse({ data: [{ id: 'm', providerID: 'p' }] }));
|
||||
global.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
await findOpencodeServer();
|
||||
const sentHeaders = fetchMock.mock.calls[0][1].headers as Record<string, string>;
|
||||
const decoded = Buffer.from(sentHeaders.Authorization.replace('Basic ', ''), 'base64').toString();
|
||||
expect(decoded).toBe('opencode:hunter2');
|
||||
});
|
||||
|
||||
it('sends no auth header at all when no password is configured', async () => {
|
||||
delete process.env.OPENCODE_SERVER_PASSWORD;
|
||||
const fetchMock = vi.fn().mockResolvedValue(jsonResponse({ data: [{ id: 'm', providerID: 'p' }] }));
|
||||
global.fetch = fetchMock as unknown as typeof fetch;
|
||||
await findOpencodeServer();
|
||||
const sentHeaders = fetchMock.mock.calls[0][1].headers as Record<string, string>;
|
||||
expect(sentHeaders.Authorization).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('findOpencodeServer', () => {
|
||||
const originalFetch = global.fetch;
|
||||
afterEach(() => {
|
||||
@@ -145,3 +180,89 @@ describe('opencodePrompt', () => {
|
||||
expect(result).toEqual({ ok: false, error: 'OpenCode returned no message content' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('listOpencodeProviders', () => {
|
||||
const originalFetch = global.fetch;
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('merges /provider and /provider/auth into one list, connected first', async () => {
|
||||
const fetchMock = vi.fn((url: string) => {
|
||||
if (url.endsWith('/provider')) {
|
||||
return Promise.resolve(jsonResponse({
|
||||
all: [{ id: 'anthropic', name: 'Anthropic' }, { id: 'deepseek', name: 'DeepSeek' }, { id: 'github-copilot', name: 'GitHub Copilot' }],
|
||||
connected: ['deepseek'],
|
||||
}));
|
||||
}
|
||||
if (url.endsWith('/provider/auth')) {
|
||||
return Promise.resolve(jsonResponse({
|
||||
anthropic: [{ type: 'api' }],
|
||||
deepseek: [{ type: 'api' }],
|
||||
'github-copilot': [{ type: 'oauth' }],
|
||||
}));
|
||||
}
|
||||
return Promise.resolve(HTML_CATCHALL);
|
||||
});
|
||||
global.fetch = fetchMock as unknown as typeof fetch;
|
||||
|
||||
const { listOpencodeProviders } = await import('../opencode');
|
||||
const result = await listOpencodeProviders('http://127.0.0.1:4096');
|
||||
|
||||
expect(result).toHaveLength(3);
|
||||
// Connected providers sort first regardless of name.
|
||||
expect(result[0]).toMatchObject({ id: 'deepseek', connected: true, supportsApiKey: true });
|
||||
const anthropic = result.find((p) => p.id === 'anthropic');
|
||||
expect(anthropic).toMatchObject({ connected: false, supportsApiKey: true });
|
||||
const copilot = result.find((p) => p.id === 'github-copilot');
|
||||
// OAuth-only provider: listed, but honestly marked as not addable here.
|
||||
expect(copilot).toMatchObject({ connected: false, supportsApiKey: false });
|
||||
});
|
||||
|
||||
it('returns an empty list rather than throwing when /provider is unreachable', async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue(HTML_CATCHALL) as unknown as typeof fetch;
|
||||
const { listOpencodeProviders } = await import('../opencode');
|
||||
expect(await listOpencodeProviders('http://127.0.0.1:4096')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('setOpencodeProviderKey / removeOpencodeProvider', () => {
|
||||
const originalFetch = global.fetch;
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('PUTs the exact schema OpenCode requires: {type:"api", key}', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true });
|
||||
global.fetch = fetchMock as unknown as typeof fetch;
|
||||
const { setOpencodeProviderKey } = await import('../opencode');
|
||||
|
||||
await setOpencodeProviderKey('http://127.0.0.1:4096', 'anthropic', 'sk-real-key');
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
'http://127.0.0.1:4096/auth/anthropic',
|
||||
expect.objectContaining({ method: 'PUT', body: JSON.stringify({ type: 'api', key: 'sk-real-key' }) }),
|
||||
);
|
||||
});
|
||||
|
||||
it('throws with the upstream status when OpenCode rejects the credential', async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 400 }) as unknown as typeof fetch;
|
||||
const { setOpencodeProviderKey } = await import('../opencode');
|
||||
await expect(setOpencodeProviderKey('http://127.0.0.1:4096', 'anthropic', 'bad')).rejects.toThrow(/400/);
|
||||
});
|
||||
|
||||
it('DELETEs by provider id and encodes it in the path', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true });
|
||||
global.fetch = fetchMock as unknown as typeof fetch;
|
||||
const { removeOpencodeProvider } = await import('../opencode');
|
||||
|
||||
await removeOpencodeProvider('http://127.0.0.1:4096', 'weird id/with slash');
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
'http://127.0.0.1:4096/auth/weird%20id%2Fwith%20slash',
|
||||
expect.objectContaining({ method: 'DELETE' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -181,6 +181,45 @@ export async function listOpencodeModels(): Promise<OpencodeModelOption[]> {
|
||||
return (body?.models ?? []) as OpencodeModelOption[];
|
||||
}
|
||||
|
||||
export interface OpencodeProviderOption {
|
||||
id: string;
|
||||
name: string;
|
||||
connected: boolean;
|
||||
supportsApiKey: boolean;
|
||||
}
|
||||
|
||||
/** Every provider OpenCode knows about, not just ones already authenticated —
|
||||
* this is what lets "add any LLM OpenCode supports" mean something from
|
||||
* inside this app instead of only whatever its CLI already set up. */
|
||||
export async function listOpencodeProviders(): Promise<OpencodeProviderOption[]> {
|
||||
const res = await fetch('/api/ai/opencode/providers');
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) throw new Error(body?.error || `OpenCode returned ${res.status}`);
|
||||
return (body?.providers ?? []) as OpencodeProviderOption[];
|
||||
}
|
||||
|
||||
/** The key is relayed to OpenCode's own credential store, never held by this
|
||||
* app — same reasoning as the module note above, extended to provider setup. */
|
||||
export async function addOpencodeProvider(providerID: string, key: string): Promise<void> {
|
||||
const res = await fetch('/api/ai/opencode/providers', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ providerID, key }),
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
// A 200 with `ok: false` means the route VERIFIED the write and the
|
||||
// provider still isn't connected (some need more than one credential
|
||||
// field — see the route's own comment) - that is as much a failure as a
|
||||
// non-2xx status and must not be swallowed as success.
|
||||
if (!res.ok || body?.ok === false) throw new Error(body?.error || `OpenCode returned ${res.status}`);
|
||||
}
|
||||
|
||||
export async function removeOpencodeProvider(providerID: string): Promise<void> {
|
||||
const res = await fetch(`/api/ai/opencode/providers?providerID=${encodeURIComponent(providerID)}`, { method: 'DELETE' });
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) throw new Error(body?.error || `OpenCode returned ${res.status}`);
|
||||
}
|
||||
|
||||
export async function chatOpencode(model: string, messages: ChatMessage[]): Promise<string> {
|
||||
const res = await fetch('/api/ai/opencode/chat', {
|
||||
method: 'POST',
|
||||
@@ -323,6 +362,15 @@ async function fetchServerLeg(question: string): Promise<{ scored: Scored<Source
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* SINGLE-MAILBOX BY POLICY (stated by the product owner 2026-08-07): the
|
||||
* assistant may only ever see the mailbox the user is currently signed in to.
|
||||
* Both legs honour that structurally rather than by filtering afterwards —
|
||||
* the local leg passes the ACTIVE account's cookie slot, and the server leg
|
||||
* resolves the same session's own JMAP account. There is deliberately no
|
||||
* fan-out across connected accounts or shared mailboxes anywhere in here, and
|
||||
* adding one later would be a policy change, not an enhancement.
|
||||
*/
|
||||
async function retrieveContext(question: string, slot?: number): Promise<RetrievedContext | null> {
|
||||
const [local, server] = await Promise.all([fetchLocalLeg(question, slot), fetchServerLeg(question)]);
|
||||
lastLocalIndexReachable = local.indexReachable;
|
||||
|
||||
+94
-1
@@ -73,11 +73,31 @@ export function parseModelRef(ref: string): { providerID: string; modelID: strin
|
||||
return { providerID: ref.slice(0, slash), modelID: ref.slice(slash + 1) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Auth header for a password-protected server.
|
||||
*
|
||||
* HTTP Basic with the username EXACTLY `opencode` — verified against 1.18.14:
|
||||
* an empty username, an arbitrary one, a Bearer token and every plausible
|
||||
* custom header all 401 with the correct password. Its own OpenAPI spec
|
||||
* declares no securitySchemes at all, so this is only knowable by trying it.
|
||||
* Absent password = an unsecured server (the desktop shell always sets one;
|
||||
* a hand-started `opencode serve` typically has none).
|
||||
*/
|
||||
function authHeaders(): Record<string, string> {
|
||||
const password = process.env.OPENCODE_SERVER_PASSWORD;
|
||||
if (!password) return {};
|
||||
return { Authorization: `Basic ${Buffer.from(`opencode:${password}`).toString('base64')}` };
|
||||
}
|
||||
|
||||
async function fetchJson(url: string, init: RequestInit, timeoutMs: number): Promise<unknown | null> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch(url, { ...init, signal: controller.signal });
|
||||
const res = await fetch(url, {
|
||||
...init,
|
||||
headers: { ...authHeaders(), ...(init.headers as Record<string, string> | undefined) },
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
// The SPA catch-all returns HTML with a 200 for unknown paths — see the
|
||||
// module header. Content-type is what actually distinguishes a real API
|
||||
@@ -111,6 +131,79 @@ export async function findOpencodeServer(): Promise<{ baseUrl: string; models: O
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Provider management ──────────────────────────────────────────────────
|
||||
//
|
||||
// Without this, "any LLM OpenCode supports" was only true for whatever the
|
||||
// user had already authenticated via its own CLI (`opencode auth login`) —
|
||||
// this app could pick a model, never add a provider. `GET /provider` lists
|
||||
// every provider opencode KNOWS about (180 on a real run) with a `connected`
|
||||
// array naming which ones actually have credentials; `GET /provider/auth`
|
||||
// says which auth METHODS each one accepts.
|
||||
//
|
||||
// Scoped to API-key auth only for now, deliberately. `PUT /auth/{id}` with
|
||||
// `{type:'api', key}` is one HTTP call with a schema-verified shape (tested
|
||||
// live: 200, and the key round-trips into opencode's own auth.json). OAuth
|
||||
// entries in `/provider/auth` (`{type:'oauth', label, prompts?}`) need a
|
||||
// browser redirect + callback this app has no page for yet, and some carry
|
||||
// interactive prompts (GitHub Copilot's deployment-type picker) beyond a
|
||||
// single form — real scope for later, not something to half-build tonight.
|
||||
// Providers offering only OAuth are still LISTED, just marked unsupported
|
||||
// here, so the picker is honest about what it can and can't do.
|
||||
|
||||
export interface OpencodeProviderInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
connected: boolean;
|
||||
/** Whether this app can authenticate it — see the module note above. */
|
||||
supportsApiKey: boolean;
|
||||
}
|
||||
|
||||
interface ProviderListResponse {
|
||||
all?: Array<{ id?: string; name?: string }>;
|
||||
connected?: string[];
|
||||
}
|
||||
|
||||
type ProviderAuthMethod = { type?: string };
|
||||
type ProviderAuthResponse = Record<string, ProviderAuthMethod[]>;
|
||||
|
||||
export async function listOpencodeProviders(baseUrl: string): Promise<OpencodeProviderInfo[]> {
|
||||
const [providers, authMethods] = await Promise.all([
|
||||
fetchJson(`${baseUrl}/provider`, {}, PROBE_TIMEOUT_MS) as Promise<ProviderListResponse | null>,
|
||||
fetchJson(`${baseUrl}/provider/auth`, {}, PROBE_TIMEOUT_MS) as Promise<ProviderAuthResponse | null>,
|
||||
]);
|
||||
if (!providers || !Array.isArray(providers.all)) return [];
|
||||
const connected = new Set(providers.connected ?? []);
|
||||
return providers.all
|
||||
.filter((p): p is { id: string; name?: string } => typeof p?.id === 'string' && !!p.id)
|
||||
.map((p) => ({
|
||||
id: p.id,
|
||||
name: p.name || p.id,
|
||||
connected: connected.has(p.id),
|
||||
supportsApiKey: (authMethods?.[p.id] ?? []).some((m) => m.type === 'api'),
|
||||
}))
|
||||
.sort((a, b) => (a.connected === b.connected ? a.name.localeCompare(b.name) : a.connected ? -1 : 1));
|
||||
}
|
||||
|
||||
/** Stores an API key for a provider. Throws with opencode's own status on
|
||||
* failure rather than returning a boolean, so the route can pass a real
|
||||
* error back instead of a bare "didn't work". */
|
||||
export async function setOpencodeProviderKey(baseUrl: string, providerID: string, key: string): Promise<void> {
|
||||
const res = await fetch(`${baseUrl}/auth/${encodeURIComponent(providerID)}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json', ...authHeaders() },
|
||||
body: JSON.stringify({ type: 'api', key }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`OpenCode rejected the credential (HTTP ${res.status})`);
|
||||
}
|
||||
|
||||
export async function removeOpencodeProvider(baseUrl: string, providerID: string): Promise<void> {
|
||||
const res = await fetch(`${baseUrl}/auth/${encodeURIComponent(providerID)}`, {
|
||||
method: 'DELETE',
|
||||
headers: authHeaders(),
|
||||
});
|
||||
if (!res.ok) throw new Error(`OpenCode could not remove the credential (HTTP ${res.status})`);
|
||||
}
|
||||
|
||||
interface OpencodeMessageResponse {
|
||||
parts?: Array<{ type?: string; text?: string }>;
|
||||
}
|
||||
|
||||
@@ -70,6 +70,9 @@ export interface IndexRequestOptions {
|
||||
ids?: Partial<Record<IndexContentType, string[]>>;
|
||||
/** Backfill the recent window for every supported type, and prune. */
|
||||
catchUp?: boolean;
|
||||
/** Retention window in days, or null to keep everything. Omitted = server
|
||||
* default (1 year). Bounds the fetch AND the prune together. */
|
||||
windowDays?: number | null;
|
||||
/** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */
|
||||
slot?: number;
|
||||
}
|
||||
@@ -96,6 +99,7 @@ export async function requestIndex(options: IndexRequestOptions = {}): Promise<I
|
||||
types: options.types,
|
||||
ids: options.ids,
|
||||
catchUp: options.catchUp === true,
|
||||
...(options.windowDays !== undefined ? { windowDays: options.windowDays } : {}),
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -169,8 +173,32 @@ export function indexOnStateChange(
|
||||
* (`client.ts`'s buildStatePollingRequest polls Mailbox/Email/Calendar/
|
||||
* CalendarEvent/SieveScript only).
|
||||
*/
|
||||
export async function catchUpIndex(slot?: number): Promise<IndexRunResult> {
|
||||
return requestIndex({ catchUp: true, slot });
|
||||
export async function catchUpIndex(slot?: number, windowDays?: number | null): Promise<IndexRunResult> {
|
||||
return requestIndex({ catchUp: true, slot, windowDays });
|
||||
}
|
||||
|
||||
// ── Retention setting ────────────────────────────────────────────────────
|
||||
// Renderer-owned: the renderer already drives every index run, so keeping the
|
||||
// choice here avoids a second source of truth on the server that could drift
|
||||
// out of step with what the user last picked.
|
||||
|
||||
const RETENTION_KEY = 'vncmail:index:retention-days';
|
||||
/** Matches DEFAULT_RETENTION_WINDOW_DAYS in lib/mail-index/reindex.ts. */
|
||||
export const DEFAULT_RETENTION_DAYS = 365;
|
||||
|
||||
/** `null` = keep everything. */
|
||||
export function getRetentionDays(): number | null {
|
||||
if (typeof window === 'undefined') return DEFAULT_RETENTION_DAYS;
|
||||
const raw = window.localStorage.getItem(RETENTION_KEY);
|
||||
if (raw === null) return DEFAULT_RETENTION_DAYS;
|
||||
if (raw === 'forever') return null;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
return Number.isFinite(parsed) ? parsed : DEFAULT_RETENTION_DAYS;
|
||||
}
|
||||
|
||||
export function setRetentionDays(days: number | null): void {
|
||||
if (typeof window === 'undefined') return;
|
||||
window.localStorage.setItem(RETENTION_KEY, days === null ? 'forever' : String(days));
|
||||
}
|
||||
|
||||
export interface IndexStats {
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { detectRecencyIntent } from '../recency';
|
||||
|
||||
// Fixed "now" so the month-name branch is deterministic: 2026-08-07.
|
||||
const NOW = new Date('2026-08-07T10:00:00.000Z');
|
||||
|
||||
describe('detectRecencyIntent', () => {
|
||||
it('recognises the two questions that actually failed against a populated index', () => {
|
||||
// Both were asked by a real user and both returned nothing useful, because
|
||||
// bm25 matched the WORDS "last"/"July" rather than the dates.
|
||||
expect(detectRecencyIntent('who sent the last email', NOW)).not.toBeNull();
|
||||
expect(detectRecencyIntent('summarize the input of all mails sent in July', NOW)).not.toBeNull();
|
||||
});
|
||||
|
||||
it('turns a month name into a bounded range, not a top-N', () => {
|
||||
const intent = detectRecencyIntent('summarize all mails sent in July', NOW);
|
||||
// Asserted in LOCAL time on purpose. The bounds are local midnight
|
||||
// rendered as UTC instants, so west of UTC the ISO string reads as the
|
||||
// previous month - and that is correct, not a bug: a mail that arrived
|
||||
// 00:30 local on 1 July belongs to the user's July even though its UTC
|
||||
// timestamp says 30 June. Asserting the ISO prefix would enshrine the
|
||||
// wrong semantics and pass only in UTC.
|
||||
const since = new Date(intent!.since!);
|
||||
const until = new Date(intent!.until!);
|
||||
expect(since.getMonth()).toBe(6); // local July...
|
||||
expect(since.getDate()).toBe(1); // ...starting on the 1st
|
||||
expect(until.getMonth()).toBe(7); // exclusive upper bound = local 1 Aug
|
||||
expect(until.getDate()).toBe(1);
|
||||
});
|
||||
|
||||
it('reads a month later in the year as LAST year', () => {
|
||||
// Asked in August, "December" cannot mean four months from now.
|
||||
const intent = detectRecencyIntent('what came in December?', NOW);
|
||||
const since = new Date(intent!.since!);
|
||||
expect(since.getFullYear()).toBe(2025);
|
||||
expect(since.getMonth()).toBe(11);
|
||||
});
|
||||
|
||||
it('handles today and yesterday as distinct bounded days', () => {
|
||||
const today = detectRecencyIntent('anything today?', NOW);
|
||||
expect(today?.since).toBeDefined();
|
||||
expect(today?.until).toBeUndefined();
|
||||
|
||||
const yesterday = detectRecencyIntent('what arrived yesterday', NOW);
|
||||
expect(yesterday?.since).toBeDefined();
|
||||
expect(yesterday?.until).toBeDefined();
|
||||
expect(new Date(yesterday!.until!).getTime()).toBeGreaterThan(new Date(yesterday!.since!).getTime());
|
||||
});
|
||||
|
||||
it('understands German recency wording — the app ships a German UI', () => {
|
||||
expect(detectRecencyIntent('welche war die letzte Mail?', NOW)).not.toBeNull();
|
||||
expect(detectRecencyIntent('was kam heute an', NOW)).not.toBeNull();
|
||||
const juli = detectRecencyIntent('Mails aus Juli zusammenfassen', NOW);
|
||||
expect(new Date(juli!.since!).getMonth()).toBe(6);
|
||||
});
|
||||
|
||||
it('gives an unbounded top-N when recency is implied but no period named', () => {
|
||||
const intent = detectRecencyIntent('what is the newest message', NOW);
|
||||
expect(intent?.since).toBeUndefined();
|
||||
expect(intent?.until).toBeUndefined();
|
||||
expect(intent?.limit).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('does NOT fire on pure content questions — those are keyword search\'s job', () => {
|
||||
expect(detectRecencyIntent('what did Anna say about the invoice?', NOW)).toBeNull();
|
||||
expect(detectRecencyIntent('when is check-in for the Villa sul Lago booking?', NOW)).toBeNull();
|
||||
expect(detectRecencyIntent('find the contract with Bechtle', NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it('does not treat a word merely CONTAINING a keyword as recency', () => {
|
||||
// "newsletter" contains "new"; "lastly" contains "last". Word boundaries
|
||||
// matter or half a mailbox reads as a time question.
|
||||
expect(detectRecencyIntent('unsubscribe from the newsletter', NOW)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -242,13 +242,17 @@ export async function queryRecentEmailIds(
|
||||
session: JmapSessionInfo,
|
||||
authHeader: string,
|
||||
accountId: string,
|
||||
afterIso: string,
|
||||
/** Lower bound, or undefined for "no date bound" (the keep-everything
|
||||
* retention choice). An `after` of undefined must be OMITTED from the
|
||||
* filter, not sent as undefined - Stalwart rejects a malformed filter
|
||||
* rather than treating it as unset. */
|
||||
afterIso: string | undefined,
|
||||
limit: number,
|
||||
): Promise<string[]> {
|
||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||
['Email/query', {
|
||||
accountId,
|
||||
filter: { after: afterIso },
|
||||
filter: afterIso ? { after: afterIso } : {},
|
||||
sort: [{ property: 'receivedAt', isAscending: false }],
|
||||
limit,
|
||||
calculateTotal: false,
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
// Recency-intent detection for the retrieval layer.
|
||||
//
|
||||
// Keyword search cannot answer a question about WHEN. bm25 ranks by term
|
||||
// overlap, so "what was the last mail I received" matches documents that
|
||||
// happen to contain the word "last", and "summarise everything from July"
|
||||
// matches documents containing "July" — not documents dated in July. Both were
|
||||
// asked by a real user against a correctly-populated index and both returned
|
||||
// nothing useful, which is what this module exists to fix: it decides when a
|
||||
// question is really a date question, and turns it into a date RANGE the index
|
||||
// can answer with an ordered scan over `occurred_at`.
|
||||
//
|
||||
// Deliberately a heuristic on English/German keywords rather than an LLM call:
|
||||
// it runs on every question, must be instant, and a false positive is cheap
|
||||
// (the recency hits are fused with the keyword hits, not substituted for them).
|
||||
|
||||
// TIMEZONE NOTE: all bounds are built from LOCAL calendar boundaries and then
|
||||
// serialised as UTC instants. That is deliberate — "July" means the user's
|
||||
// July, so a mail received 00:30 local on 1 July belongs to it even though its
|
||||
// stored UTC timestamp reads 30 June. Building the bounds in UTC instead would
|
||||
// silently drop the first/last hours of every named period for anyone not on
|
||||
// UTC.
|
||||
export interface RecencyIntent {
|
||||
/** ISO lower bound, if the question named one. */
|
||||
since?: string;
|
||||
/** ISO upper bound, if the question named a closed period. */
|
||||
until?: string;
|
||||
/** How many documents the recency leg should contribute. */
|
||||
limit: number;
|
||||
}
|
||||
|
||||
const RECENCY_WORDS = [
|
||||
// English
|
||||
'last', 'latest', 'recent', 'recently', 'newest', 'new', 'today', 'yesterday',
|
||||
'this week', 'this month', 'past week', 'past month', 'so far', 'just now', 'current',
|
||||
// German — the app ships a German UI and users mix languages freely
|
||||
'letzte', 'letzten', 'letzter', 'neueste', 'neuesten', 'neu', 'heute', 'gestern',
|
||||
'diese woche', 'diesen monat', 'kürzlich', 'zuletzt', 'aktuell',
|
||||
];
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
january: 0, february: 1, march: 2, april: 3, may: 4, june: 5,
|
||||
july: 6, august: 7, september: 8, october: 9, november: 10, december: 11,
|
||||
januar: 0, februar: 1, märz: 2, maerz: 2, mai: 4, juni: 5,
|
||||
juli: 6, oktober: 9, dezember: 11,
|
||||
};
|
||||
|
||||
function startOfDay(d: Date): Date {
|
||||
const c = new Date(d);
|
||||
c.setHours(0, 0, 0, 0);
|
||||
return c;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param now injected so the behaviour is testable and deterministic — the
|
||||
* month-name branch depends on "which year is it" and must not be a coin
|
||||
* flip in a test suite.
|
||||
*/
|
||||
export function detectRecencyIntent(question: string, now: Date = new Date()): RecencyIntent | null {
|
||||
const q = question.toLowerCase();
|
||||
|
||||
// A named month wins over generic recency words: "everything from July" is a
|
||||
// bounded range, which is far more useful than "the newest N".
|
||||
for (const [name, monthIndex] of Object.entries(MONTHS)) {
|
||||
if (!new RegExp(`\\b${name}\\b`).test(q)) continue;
|
||||
// A month later than the current one must mean LAST year — "July" asked in
|
||||
// March means the July that already happened, not one nine months away.
|
||||
const year = monthIndex > now.getMonth() ? now.getFullYear() - 1 : now.getFullYear();
|
||||
const since = new Date(year, monthIndex, 1, 0, 0, 0, 0);
|
||||
const until = new Date(year, monthIndex + 1, 1, 0, 0, 0, 0);
|
||||
return { since: since.toISOString(), until: until.toISOString(), limit: 40 };
|
||||
}
|
||||
|
||||
if (/\btoday\b|\bheute\b/.test(q)) {
|
||||
return { since: startOfDay(now).toISOString(), limit: 25 };
|
||||
}
|
||||
if (/\byesterday\b|\bgestern\b/.test(q)) {
|
||||
const start = startOfDay(new Date(now.getTime() - 86_400_000));
|
||||
return { since: start.toISOString(), until: startOfDay(now).toISOString(), limit: 25 };
|
||||
}
|
||||
if (/this week|past week|diese woche|letzte woche/.test(q)) {
|
||||
return { since: startOfDay(new Date(now.getTime() - 7 * 86_400_000)).toISOString(), limit: 40 };
|
||||
}
|
||||
if (/this month|past month|diesen monat|letzten monat/.test(q)) {
|
||||
return { since: startOfDay(new Date(now.getTime() - 30 * 86_400_000)).toISOString(), limit: 40 };
|
||||
}
|
||||
|
||||
if (RECENCY_WORDS.some((w) => (w.includes(' ') ? q.includes(w) : new RegExp(`\\b${w}\\b`).test(q)))) {
|
||||
// No period named — "the last mail", "what's new". Unbounded top-N.
|
||||
return { limit: 15 };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
+62
-13
@@ -29,16 +29,51 @@ import { withIndexKey } from './key';
|
||||
import { getStoreDir } from './paths';
|
||||
import { MailIndex, type ContentType, type IndexDoc } from './store';
|
||||
|
||||
/**
|
||||
* Bounded window. Small on purpose: this is the first cut of a retrieval index,
|
||||
* and a wide window turns "index on every delivery" into a slow request. The
|
||||
* event-driven path indexes single objects, so the window only bounds catch-up.
|
||||
*/
|
||||
export const INDEX_WINDOW_DAYS = 30;
|
||||
/** Calendar looks forward as well as back - upcoming events are the useful ones. */
|
||||
export const CALENDAR_FORWARD_DAYS = 180;
|
||||
/** Per-type ceiling for one catch-up pass. */
|
||||
/** Per-type ceiling for one catch-up pass, per 30 days of window. */
|
||||
export const CATCHUP_MAX_PER_TYPE = 500;
|
||||
|
||||
/**
|
||||
* How much mail history the local index keeps. User-selectable
|
||||
* (Settings -> About & Data); `null` means keep everything and never prune.
|
||||
*
|
||||
* This is NOT just a fetch bound - catch-up also PRUNES mail older than it.
|
||||
* The original hardcoded 30 days therefore made a question like "summarise
|
||||
* everything from July" unanswerable in August: the rows had been deliberately
|
||||
* deleted, while the UI said only that nothing matched. A real user hit exactly
|
||||
* that, which is why this is a setting with a year-long default rather than a
|
||||
* constant tuned for a first cut.
|
||||
*/
|
||||
export const RETENTION_CHOICES = [30, 90, 365, null] as const;
|
||||
export type RetentionWindowDays = (typeof RETENTION_CHOICES)[number];
|
||||
export const DEFAULT_RETENTION_WINDOW_DAYS: RetentionWindowDays = 365;
|
||||
|
||||
/** Hard ceiling regardless of window - one pass must still terminate. */
|
||||
export const CATCHUP_MAX_PER_TYPE_UNLIMITED = 20_000;
|
||||
|
||||
export function normalizeWindowDays(raw: unknown): RetentionWindowDays {
|
||||
if (raw === null) return null;
|
||||
if (typeof raw !== 'number' || !Number.isFinite(raw)) return DEFAULT_RETENTION_WINDOW_DAYS;
|
||||
// Anything off the list falls back to the default rather than being honoured
|
||||
// verbatim - this value drives DELETION, so a typo'd 0 must never silently
|
||||
// wipe the index.
|
||||
const allowed: readonly number[] = RETENTION_CHOICES.filter((c) => c !== null);
|
||||
return allowed.includes(raw) ? (raw as RetentionWindowDays) : DEFAULT_RETENTION_WINDOW_DAYS;
|
||||
}
|
||||
|
||||
/** Scales the per-pass ceiling with the window: 500 is right for a month and
|
||||
* nonsense for "everything", where having the history IS the point. */
|
||||
export function catchUpCapFor(windowDays: RetentionWindowDays): number {
|
||||
if (windowDays === null) return CATCHUP_MAX_PER_TYPE_UNLIMITED;
|
||||
return Math.min(CATCHUP_MAX_PER_TYPE_UNLIMITED, Math.round((windowDays / 30) * CATCHUP_MAX_PER_TYPE));
|
||||
}
|
||||
|
||||
/** Lower bound for a date-filtered query, or undefined when unlimited. */
|
||||
export function windowStartIso(windowDays: RetentionWindowDays): string | undefined {
|
||||
return windowDays === null ? undefined : isoDaysFromNow(-windowDays);
|
||||
}
|
||||
|
||||
/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
|
||||
export const MAX_IDS_PER_CALL = 200;
|
||||
/** Cap on body bytes requested per message from the server. */
|
||||
@@ -169,6 +204,7 @@ interface FetchArgs {
|
||||
authHeader: string;
|
||||
jmapAccountId: string;
|
||||
ids: readonly string[] | null;
|
||||
windowDays: RetentionWindowDays;
|
||||
}
|
||||
|
||||
interface FetchResult {
|
||||
@@ -184,13 +220,14 @@ interface FetchResult {
|
||||
|
||||
/** Fetches and flattens one content type. `ids === null` means "the recent window". */
|
||||
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<FetchResult> {
|
||||
const { session, authHeader, jmapAccountId, ids } = args;
|
||||
const { session, authHeader, jmapAccountId, ids, windowDays } = args;
|
||||
const cap = catchUpCapFor(windowDays);
|
||||
|
||||
switch (contentType) {
|
||||
case 'mail': {
|
||||
const targetIds = ids ?? await queryRecentEmailIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE,
|
||||
windowStartIso(windowDays), cap,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
// Chunked because bodies are big: one Email/get for 500 messages with
|
||||
@@ -206,8 +243,11 @@ async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<Fet
|
||||
case 'calendar': {
|
||||
const targetIds = ids ?? await queryCalendarEventIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||
CATCHUP_MAX_PER_TYPE,
|
||||
// Calendar keeps its own bounded look-back even under "keep
|
||||
// everything": events are small but a decade of them is noise in a
|
||||
// mail assistant's context, and the forward window is the useful half.
|
||||
windowStartIso(windowDays) ?? isoDaysFromNow(-365), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||
cap,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 50) {
|
||||
@@ -260,6 +300,10 @@ export interface IndexRequest {
|
||||
removed?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Drop documents outside the retention window after writing. */
|
||||
prune?: boolean;
|
||||
/** Retention window in days, or null to keep everything. Bounds BOTH the
|
||||
* catch-up fetch and the prune, so the two can never disagree and delete
|
||||
* what was just written. Defaults to DEFAULT_RETENTION_WINDOW_DAYS. */
|
||||
windowDays?: RetentionWindowDays;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -274,6 +318,7 @@ export async function runIndex(
|
||||
req: IndexRequest,
|
||||
): Promise<IndexResult> {
|
||||
const started = Date.now();
|
||||
const windowDays = normalizeWindowDays(req.windowDays === undefined ? DEFAULT_RETENTION_WINDOW_DAYS : req.windowDays);
|
||||
const storeDir = getStoreDir();
|
||||
if (!storeDir) {
|
||||
throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
|
||||
@@ -325,14 +370,18 @@ export async function runIndex(
|
||||
: null;
|
||||
|
||||
const { docs, queriedCount } = await fetchDocs(contentType, {
|
||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids,
|
||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids, windowDays,
|
||||
});
|
||||
written[contentType] = index.upsert(docs);
|
||||
|
||||
if (req.prune && contentType === 'mail') {
|
||||
// Only mail prunes by date: calendar's window looks forward,
|
||||
// contacts have no date, and file rows are metadata-sized.
|
||||
index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS));
|
||||
// A null window means keep everything - pruning is SKIPPED, not
|
||||
// run with some fallback bound, or the setting would silently
|
||||
// delete the history the user just asked to retain.
|
||||
const cutoff = windowStartIso(windowDays);
|
||||
if (cutoff) index.pruneOlderThan(jmapAccountId, 'mail', cutoff);
|
||||
}
|
||||
|
||||
// Contact/file DELETES: a JMAP `destroyed` only ever reaches this
|
||||
|
||||
@@ -374,6 +374,69 @@ export class MailIndex {
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Newest documents by date, ignoring keyword relevance entirely.
|
||||
*
|
||||
* The retrieval leg for RECENCY questions — "what was the last mail", "who
|
||||
* wrote most recently", "everything from July". Full-text search cannot
|
||||
* answer those even with a perfect index: bm25 ranks by term overlap and has
|
||||
* no notion of "latest", so "the last mail" matches documents containing the
|
||||
* word "last". Two real questions failed exactly that way before this
|
||||
* existed. `doc(jmap_account_id, content_type, occurred_at DESC)` is already
|
||||
* indexed, so this is an ordered range scan, not a table sweep.
|
||||
*
|
||||
* `since`/`until` are ISO strings, both optional — a month-name question
|
||||
* becomes a bounded range, a bare "latest" becomes an unbounded top-N.
|
||||
*/
|
||||
recent(opts: {
|
||||
types?: readonly ContentType[];
|
||||
limit?: number;
|
||||
since?: string;
|
||||
until?: string;
|
||||
snippetChars?: number;
|
||||
}): SearchHit[] {
|
||||
const limit = Math.min(Math.max(opts.limit ?? 10, 1), 200);
|
||||
const snippetChars = Math.min(Math.max(opts.snippetChars ?? 400, 80), 2000);
|
||||
const types = opts.types && opts.types.length > 0 ? opts.types : null;
|
||||
|
||||
const where: string[] = ['d.occurred_at IS NOT NULL'];
|
||||
const params: Array<string | number> = [];
|
||||
if (types) {
|
||||
where.push(`d.content_type IN (${types.map(() => '?').join(',')})`);
|
||||
params.push(...types);
|
||||
}
|
||||
if (opts.since) { where.push('d.occurred_at >= ?'); params.push(opts.since); }
|
||||
if (opts.until) { where.push('d.occurred_at <= ?'); params.push(opts.until); }
|
||||
|
||||
const rows = this.db
|
||||
.prepare(`
|
||||
SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people,
|
||||
d.occurred_at, d.metadata_json,
|
||||
substr(f.body, 1, ${snippetChars}) AS snip
|
||||
FROM doc d
|
||||
LEFT JOIN doc_fts f ON f.rowid = d.rowid
|
||||
WHERE ${where.join(' AND ')}
|
||||
ORDER BY d.occurred_at DESC
|
||||
LIMIT ?
|
||||
`)
|
||||
.all([...params, limit]);
|
||||
|
||||
return rows.map((r) => ({
|
||||
contentType: String(r.content_type) as ContentType,
|
||||
id: String(r.id),
|
||||
jmapAccountId: String(r.jmap_account_id),
|
||||
title: String(r.title ?? ''),
|
||||
people: String(r.people ?? ''),
|
||||
occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at),
|
||||
metadata: safeParseObject(r.metadata_json),
|
||||
// No bm25 score here: these are ordered by time, not relevance, and
|
||||
// faking a relevance number would let the fusion step rank them as if
|
||||
// they had been scored.
|
||||
score: 0,
|
||||
snippet: String(r.snip ?? ''),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Per-type counts and freshness, for the Settings UI and for debugging. */
|
||||
stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> {
|
||||
return this.db
|
||||
|
||||
Reference in New Issue
Block a user