feat(mail-index): 1-year retention by default + a recency retrieval leg
The two things that made real questions fail against a correctly-populated index, both fixed at the root. RETENTION (A1). `INDEX_WINDOW_DAYS = 30` was not merely a fetch bound — catch-up also PRUNED mail older than it, so "summarise everything from July" was unanswerable in August because the rows had been deleted, while the UI said only that nothing matched. Now a user-visible setting (Settings → About & Data): 30 days / 3 months / 1 year / everything, defaulting to 1 YEAR per the product owner. The window bounds the fetch AND the prune from one value so the two can never disagree and delete what was just written; "everything" skips pruning entirely rather than falling back to some default bound. The per-pass ceiling scales with the window (500/30d, hard cap 20k) because 500 messages is right for a month and nonsense for "everything". Email/query now omits the `after` filter entirely when unbounded — Stalwart rejects a malformed filter rather than treating `undefined` as unset. RECENCY (A2). Keyword search structurally cannot answer a question about WHEN: bm25 ranks by term overlap, so "who sent the last email" matches documents containing the word "last", and "all mails in July" matches documents containing "July" — not documents dated in July. Both were asked by a real user and both failed. New lib/mail-index/recency.ts detects time intent (English + German, since the UI ships German) and turns it into a date RANGE; new MailIndex.recent() answers it with an ordered scan over the already-indexed `occurred_at`. The route ADDS these hits to the keyword hits rather than replacing them — "what did the last mail from Anna say" is both kinds of question at once. Timezone subtlety worth knowing: bounds are built from LOCAL calendar boundaries and serialised as UTC instants, so "July" covers the user's July. A mail at 00:30 local on 1 July belongs to it even though its stored UTC timestamp reads 30 June. My first test asserted the ISO string prefix, which would have enshrined the opposite and passed only in UTC — the tests now assert the local-time property instead. SCOPE, stated by the product owner and now enforced structurally: the assistant only ever sees the mailbox the user is signed in to. Both retrieval legs resolve the active account (local leg by cookie slot, server leg by the session's own JMAP account); there is deliberately no fan-out across connected or shared mailboxes, and adding one would be a policy change, not a feature. Gate: tsc clean, eslint clean, 2520/2520 tests (8 new for recency intent), build clean.
This commit is contained in:
@@ -1088,9 +1088,10 @@ export default function Home() {
|
|||||||
const runCatchUp = async (attempt: number) => {
|
const runCatchUp = async (attempt: number) => {
|
||||||
if (catchUpCancelled) return;
|
if (catchUpCancelled) return;
|
||||||
try {
|
try {
|
||||||
const { catchUpIndex } = await import('@/lib/mail-index-client');
|
const { catchUpIndex, getRetentionDays } = await import('@/lib/mail-index-client');
|
||||||
const result = await catchUpIndex(
|
const result = await catchUpIndex(
|
||||||
useAccountStore.getState().getActiveAccount()?.cookieSlot,
|
useAccountStore.getState().getActiveAccount()?.cookieSlot,
|
||||||
|
getRetentionDays(),
|
||||||
);
|
);
|
||||||
if (!result.ok && !result.unavailable && attempt + 1 < catchUpRetryDelaysMs.length) {
|
if (!result.ok && !result.unavailable && attempt + 1 < catchUpRetryDelaysMs.length) {
|
||||||
catchUpTimer = setTimeout(() => void runCatchUp(attempt + 1), catchUpRetryDelaysMs[attempt + 1]);
|
catchUpTimer = setTimeout(() => void runCatchUp(attempt + 1), catchUpRetryDelaysMs[attempt + 1]);
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import { isSqlcipherAvailable } from '@/lib/mail-index/binding';
|
|||||||
import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key';
|
import { hasKeyChannel, IndexKeyError } from '@/lib/mail-index/key';
|
||||||
import { getStoreDir } from '@/lib/mail-index/paths';
|
import { getStoreDir } from '@/lib/mail-index/paths';
|
||||||
import {
|
import {
|
||||||
IndexSessionError, MAX_IDS_PER_CALL, resolveIndexSession, runIndex,
|
IndexSessionError, MAX_IDS_PER_CALL, normalizeWindowDays, resolveIndexSession, runIndex,
|
||||||
type IndexRequest,
|
type IndexRequest,
|
||||||
} from '@/lib/mail-index/reindex';
|
} from '@/lib/mail-index/reindex';
|
||||||
import { CONTENT_TYPES, isContentType, type ContentType } from '@/lib/mail-index/store';
|
import { CONTENT_TYPES, isContentType, type ContentType } from '@/lib/mail-index/store';
|
||||||
@@ -73,6 +73,10 @@ export async function POST(request: NextRequest) {
|
|||||||
removed: parseIdMap(body.removed),
|
removed: parseIdMap(body.removed),
|
||||||
// Pruning is a catch-up concern; a single-delivery call shouldn't scan.
|
// Pruning is a catch-up concern; a single-delivery call shouldn't scan.
|
||||||
prune: body.catchUp === true,
|
prune: body.catchUp === true,
|
||||||
|
// `undefined` (absent) means "use the default"; an explicit null means
|
||||||
|
// keep everything. normalizeWindowDays() in runIndex clamps anything
|
||||||
|
// unexpected, since this value drives deletion.
|
||||||
|
windowDays: body.windowDays === undefined ? undefined : normalizeWindowDays(body.windowDays),
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import { IndexSessionError, resolveIndexSession } from '@/lib/mail-index/reindex
|
|||||||
import {
|
import {
|
||||||
isContentType, MailIndex, MailIndexUnavailableError, type ContentType, type SearchHit,
|
isContentType, MailIndex, MailIndexUnavailableError, type ContentType, type SearchHit,
|
||||||
} from '@/lib/mail-index/store';
|
} from '@/lib/mail-index/store';
|
||||||
|
import { detectRecencyIntent } from '@/lib/mail-index/recency';
|
||||||
|
|
||||||
export const runtime = 'nodejs';
|
export const runtime = 'nodejs';
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
@@ -84,7 +85,29 @@ export async function GET(request: NextRequest) {
|
|||||||
// not deliberate search-box keywords, so strict AND-every-token
|
// not deliberate search-box keywords, so strict AND-every-token
|
||||||
// matching (the default) drops nearly all of them. See
|
// matching (the default) drops nearly all of them. See
|
||||||
// toFtsMatchQueryAny's docstring for the confirmed-live failure.
|
// toFtsMatchQueryAny's docstring for the confirmed-live failure.
|
||||||
return { hits: index.search({ query, types, limit, mode: 'any' }), stats: wantStats ? stats : undefined };
|
const keywordHits = index.search({ query, types, limit, mode: 'any' });
|
||||||
|
|
||||||
|
// RECENCY leg. Keyword search structurally cannot answer "the last
|
||||||
|
// mail" or "everything from July" (see lib/mail-index/recency.ts), so
|
||||||
|
// when the question is really about time, add a date-ordered slice.
|
||||||
|
// ADDED to the keyword hits rather than replacing them: "what did the
|
||||||
|
// last mail from Anna say" is both a time question and a content one.
|
||||||
|
const intent = detectRecencyIntent(query);
|
||||||
|
if (!intent) {
|
||||||
|
return { hits: keywordHits, stats: wantStats ? stats : undefined };
|
||||||
|
}
|
||||||
|
const recentHits = index.recent({
|
||||||
|
types, limit: Math.min(intent.limit, limit * 3), since: intent.since, until: intent.until,
|
||||||
|
});
|
||||||
|
const seen = new Set(keywordHits.map((h) => `${h.contentType}:${h.id}`));
|
||||||
|
const merged = [...keywordHits];
|
||||||
|
for (const hit of recentHits) {
|
||||||
|
const key = `${hit.contentType}:${hit.id}`;
|
||||||
|
if (seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
merged.push(hit);
|
||||||
|
}
|
||||||
|
return { hits: merged, stats: wantStats ? stats : undefined, recency: intent };
|
||||||
} finally {
|
} finally {
|
||||||
index.close();
|
index.close();
|
||||||
}
|
}
|
||||||
@@ -100,6 +123,10 @@ export async function GET(request: NextRequest) {
|
|||||||
// Everything a prompt needs, pre-joined in rank order.
|
// Everything a prompt needs, pre-joined in rank order.
|
||||||
contextBlock: payload.hits.map(toContextBlock).join('\n\n---\n\n'),
|
contextBlock: payload.hits.map(toContextBlock).join('\n\n---\n\n'),
|
||||||
...(payload.stats ? { stats: payload.stats } : {}),
|
...(payload.stats ? { stats: payload.stats } : {}),
|
||||||
|
// Present when the question was read as a time question — lets the
|
||||||
|
// client say "these are the newest N" instead of implying relevance
|
||||||
|
// ranking it did not do.
|
||||||
|
...(payload.recency ? { recency: payload.recency } : {}),
|
||||||
},
|
},
|
||||||
{ headers: { 'Cache-Control': 'no-store' } },
|
{ headers: { 'Cache-Control': 'no-store' } },
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -14,7 +14,9 @@ import { Button } from '@/components/ui/button';
|
|||||||
import { SettingsSection, SettingItem } from './settings-section';
|
import { SettingsSection, SettingItem } from './settings-section';
|
||||||
import { isElectronShell } from '@/lib/electron-bridge';
|
import { isElectronShell } from '@/lib/electron-bridge';
|
||||||
import { useAccountStore } from '@/stores/account-store';
|
import { useAccountStore } from '@/stores/account-store';
|
||||||
import { catchUpIndex, fetchIndexStats, type IndexStats } from '@/lib/mail-index-client';
|
import {
|
||||||
|
catchUpIndex, fetchIndexStats, getRetentionDays, setRetentionDays, type IndexStats,
|
||||||
|
} from '@/lib/mail-index-client';
|
||||||
import {
|
import {
|
||||||
chainSync, fetchReplicaStatus, purgeReplica, updateRetentionPolicy,
|
chainSync, fetchReplicaStatus, purgeReplica, updateRetentionPolicy,
|
||||||
type ReplicaStatus, type RetentionPolicy,
|
type ReplicaStatus, type RetentionPolicy,
|
||||||
@@ -35,6 +37,10 @@ export function LocalIndexSettings() {
|
|||||||
// `null` until the first probe resolves, so we don't flash a panel that then
|
// `null` until the first probe resolves, so we don't flash a panel that then
|
||||||
// vanishes on a non-desktop build.
|
// vanishes on a non-desktop build.
|
||||||
const [available, setAvailable] = useState<boolean | null>(null);
|
const [available, setAvailable] = useState<boolean | null>(null);
|
||||||
|
// `null` = keep everything. Read once on mount; the setter writes through.
|
||||||
|
const [retentionDays, setRetentionDaysState] = useState<number | null>(365);
|
||||||
|
|
||||||
|
useEffect(() => { setRetentionDaysState(getRetentionDays()); }, []);
|
||||||
|
|
||||||
const refreshStats = useCallback(async () => {
|
const refreshStats = useCallback(async () => {
|
||||||
const next = await fetchIndexStats(slot);
|
const next = await fetchIndexStats(slot);
|
||||||
@@ -54,7 +60,7 @@ export function LocalIndexSettings() {
|
|||||||
setBusy(true);
|
setBusy(true);
|
||||||
setMessage(null);
|
setMessage(null);
|
||||||
try {
|
try {
|
||||||
const result = await catchUpIndex(slot);
|
const result = await catchUpIndex(slot, retentionDays);
|
||||||
if (result.unavailable) {
|
if (result.unavailable) {
|
||||||
setAvailable(false);
|
setAvailable(false);
|
||||||
setMessage(result.error ?? 'The encrypted index is unavailable on this system.');
|
setMessage(result.error ?? 'The encrypted index is unavailable on this system.');
|
||||||
@@ -110,6 +116,32 @@ export function LocalIndexSettings() {
|
|||||||
<span className="text-sm text-muted-foreground tabular-nums">{total}</span>
|
<span className="text-sm text-muted-foreground tabular-nums">{total}</span>
|
||||||
</SettingItem>
|
</SettingItem>
|
||||||
|
|
||||||
|
<SettingItem
|
||||||
|
label="Keep AI search history for"
|
||||||
|
description={
|
||||||
|
'How far back the AI assistant can search your mail. This also PRUNES: mail older ' +
|
||||||
|
'than the window is removed from the local index on the next update, so a short ' +
|
||||||
|
'window means questions about older mail cannot be answered. Only ever covers the ' +
|
||||||
|
'mailbox you are signed in to.'
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<select
|
||||||
|
className="h-9 rounded-md border border-border bg-background px-2 text-sm"
|
||||||
|
value={retentionDays === null ? 'forever' : String(retentionDays)}
|
||||||
|
onChange={(e) => {
|
||||||
|
const next = e.target.value === 'forever' ? null : Number.parseInt(e.target.value, 10);
|
||||||
|
setRetentionDaysState(next);
|
||||||
|
setRetentionDays(next);
|
||||||
|
setMessage('Saved. Choose "Update index" to apply it now.');
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<option value="30">30 days</option>
|
||||||
|
<option value="90">3 months</option>
|
||||||
|
<option value="365">1 year</option>
|
||||||
|
<option value="forever">Everything</option>
|
||||||
|
</select>
|
||||||
|
</SettingItem>
|
||||||
|
|
||||||
<SettingItem
|
<SettingItem
|
||||||
label="Update now"
|
label="Update now"
|
||||||
description={
|
description={
|
||||||
|
|||||||
@@ -323,6 +323,15 @@ async function fetchServerLeg(question: string): Promise<{ scored: Scored<Source
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SINGLE-MAILBOX BY POLICY (stated by the product owner 2026-08-07): the
|
||||||
|
* assistant may only ever see the mailbox the user is currently signed in to.
|
||||||
|
* Both legs honour that structurally rather than by filtering afterwards —
|
||||||
|
* the local leg passes the ACTIVE account's cookie slot, and the server leg
|
||||||
|
* resolves the same session's own JMAP account. There is deliberately no
|
||||||
|
* fan-out across connected accounts or shared mailboxes anywhere in here, and
|
||||||
|
* adding one later would be a policy change, not an enhancement.
|
||||||
|
*/
|
||||||
async function retrieveContext(question: string, slot?: number): Promise<RetrievedContext | null> {
|
async function retrieveContext(question: string, slot?: number): Promise<RetrievedContext | null> {
|
||||||
const [local, server] = await Promise.all([fetchLocalLeg(question, slot), fetchServerLeg(question)]);
|
const [local, server] = await Promise.all([fetchLocalLeg(question, slot), fetchServerLeg(question)]);
|
||||||
lastLocalIndexReachable = local.indexReachable;
|
lastLocalIndexReachable = local.indexReachable;
|
||||||
|
|||||||
@@ -70,6 +70,9 @@ export interface IndexRequestOptions {
|
|||||||
ids?: Partial<Record<IndexContentType, string[]>>;
|
ids?: Partial<Record<IndexContentType, string[]>>;
|
||||||
/** Backfill the recent window for every supported type, and prune. */
|
/** Backfill the recent window for every supported type, and prune. */
|
||||||
catchUp?: boolean;
|
catchUp?: boolean;
|
||||||
|
/** Retention window in days, or null to keep everything. Omitted = server
|
||||||
|
* default (1 year). Bounds the fetch AND the prune together. */
|
||||||
|
windowDays?: number | null;
|
||||||
/** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */
|
/** Cookie slot of the account to index. Defaults to the server's first signed-in slot. */
|
||||||
slot?: number;
|
slot?: number;
|
||||||
}
|
}
|
||||||
@@ -96,6 +99,7 @@ export async function requestIndex(options: IndexRequestOptions = {}): Promise<I
|
|||||||
types: options.types,
|
types: options.types,
|
||||||
ids: options.ids,
|
ids: options.ids,
|
||||||
catchUp: options.catchUp === true,
|
catchUp: options.catchUp === true,
|
||||||
|
...(options.windowDays !== undefined ? { windowDays: options.windowDays } : {}),
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -169,8 +173,32 @@ export function indexOnStateChange(
|
|||||||
* (`client.ts`'s buildStatePollingRequest polls Mailbox/Email/Calendar/
|
* (`client.ts`'s buildStatePollingRequest polls Mailbox/Email/Calendar/
|
||||||
* CalendarEvent/SieveScript only).
|
* CalendarEvent/SieveScript only).
|
||||||
*/
|
*/
|
||||||
export async function catchUpIndex(slot?: number): Promise<IndexRunResult> {
|
export async function catchUpIndex(slot?: number, windowDays?: number | null): Promise<IndexRunResult> {
|
||||||
return requestIndex({ catchUp: true, slot });
|
return requestIndex({ catchUp: true, slot, windowDays });
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Retention setting ────────────────────────────────────────────────────
|
||||||
|
// Renderer-owned: the renderer already drives every index run, so keeping the
|
||||||
|
// choice here avoids a second source of truth on the server that could drift
|
||||||
|
// out of step with what the user last picked.
|
||||||
|
|
||||||
|
const RETENTION_KEY = 'vncmail:index:retention-days';
|
||||||
|
/** Matches DEFAULT_RETENTION_WINDOW_DAYS in lib/mail-index/reindex.ts. */
|
||||||
|
export const DEFAULT_RETENTION_DAYS = 365;
|
||||||
|
|
||||||
|
/** `null` = keep everything. */
|
||||||
|
export function getRetentionDays(): number | null {
|
||||||
|
if (typeof window === 'undefined') return DEFAULT_RETENTION_DAYS;
|
||||||
|
const raw = window.localStorage.getItem(RETENTION_KEY);
|
||||||
|
if (raw === null) return DEFAULT_RETENTION_DAYS;
|
||||||
|
if (raw === 'forever') return null;
|
||||||
|
const parsed = Number.parseInt(raw, 10);
|
||||||
|
return Number.isFinite(parsed) ? parsed : DEFAULT_RETENTION_DAYS;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function setRetentionDays(days: number | null): void {
|
||||||
|
if (typeof window === 'undefined') return;
|
||||||
|
window.localStorage.setItem(RETENTION_KEY, days === null ? 'forever' : String(days));
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IndexStats {
|
export interface IndexStats {
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { detectRecencyIntent } from '../recency';
|
||||||
|
|
||||||
|
// Fixed "now" so the month-name branch is deterministic: 2026-08-07.
|
||||||
|
const NOW = new Date('2026-08-07T10:00:00.000Z');
|
||||||
|
|
||||||
|
describe('detectRecencyIntent', () => {
|
||||||
|
it('recognises the two questions that actually failed against a populated index', () => {
|
||||||
|
// Both were asked by a real user and both returned nothing useful, because
|
||||||
|
// bm25 matched the WORDS "last"/"July" rather than the dates.
|
||||||
|
expect(detectRecencyIntent('who sent the last email', NOW)).not.toBeNull();
|
||||||
|
expect(detectRecencyIntent('summarize the input of all mails sent in July', NOW)).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('turns a month name into a bounded range, not a top-N', () => {
|
||||||
|
const intent = detectRecencyIntent('summarize all mails sent in July', NOW);
|
||||||
|
// Asserted in LOCAL time on purpose. The bounds are local midnight
|
||||||
|
// rendered as UTC instants, so west of UTC the ISO string reads as the
|
||||||
|
// previous month - and that is correct, not a bug: a mail that arrived
|
||||||
|
// 00:30 local on 1 July belongs to the user's July even though its UTC
|
||||||
|
// timestamp says 30 June. Asserting the ISO prefix would enshrine the
|
||||||
|
// wrong semantics and pass only in UTC.
|
||||||
|
const since = new Date(intent!.since!);
|
||||||
|
const until = new Date(intent!.until!);
|
||||||
|
expect(since.getMonth()).toBe(6); // local July...
|
||||||
|
expect(since.getDate()).toBe(1); // ...starting on the 1st
|
||||||
|
expect(until.getMonth()).toBe(7); // exclusive upper bound = local 1 Aug
|
||||||
|
expect(until.getDate()).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reads a month later in the year as LAST year', () => {
|
||||||
|
// Asked in August, "December" cannot mean four months from now.
|
||||||
|
const intent = detectRecencyIntent('what came in December?', NOW);
|
||||||
|
const since = new Date(intent!.since!);
|
||||||
|
expect(since.getFullYear()).toBe(2025);
|
||||||
|
expect(since.getMonth()).toBe(11);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('handles today and yesterday as distinct bounded days', () => {
|
||||||
|
const today = detectRecencyIntent('anything today?', NOW);
|
||||||
|
expect(today?.since).toBeDefined();
|
||||||
|
expect(today?.until).toBeUndefined();
|
||||||
|
|
||||||
|
const yesterday = detectRecencyIntent('what arrived yesterday', NOW);
|
||||||
|
expect(yesterday?.since).toBeDefined();
|
||||||
|
expect(yesterday?.until).toBeDefined();
|
||||||
|
expect(new Date(yesterday!.until!).getTime()).toBeGreaterThan(new Date(yesterday!.since!).getTime());
|
||||||
|
});
|
||||||
|
|
||||||
|
it('understands German recency wording — the app ships a German UI', () => {
|
||||||
|
expect(detectRecencyIntent('welche war die letzte Mail?', NOW)).not.toBeNull();
|
||||||
|
expect(detectRecencyIntent('was kam heute an', NOW)).not.toBeNull();
|
||||||
|
const juli = detectRecencyIntent('Mails aus Juli zusammenfassen', NOW);
|
||||||
|
expect(new Date(juli!.since!).getMonth()).toBe(6);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('gives an unbounded top-N when recency is implied but no period named', () => {
|
||||||
|
const intent = detectRecencyIntent('what is the newest message', NOW);
|
||||||
|
expect(intent?.since).toBeUndefined();
|
||||||
|
expect(intent?.until).toBeUndefined();
|
||||||
|
expect(intent?.limit).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does NOT fire on pure content questions — those are keyword search\'s job', () => {
|
||||||
|
expect(detectRecencyIntent('what did Anna say about the invoice?', NOW)).toBeNull();
|
||||||
|
expect(detectRecencyIntent('when is check-in for the Villa sul Lago booking?', NOW)).toBeNull();
|
||||||
|
expect(detectRecencyIntent('find the contract with Bechtle', NOW)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not treat a word merely CONTAINING a keyword as recency', () => {
|
||||||
|
// "newsletter" contains "new"; "lastly" contains "last". Word boundaries
|
||||||
|
// matter or half a mailbox reads as a time question.
|
||||||
|
expect(detectRecencyIntent('unsubscribe from the newsletter', NOW)).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -242,13 +242,17 @@ export async function queryRecentEmailIds(
|
|||||||
session: JmapSessionInfo,
|
session: JmapSessionInfo,
|
||||||
authHeader: string,
|
authHeader: string,
|
||||||
accountId: string,
|
accountId: string,
|
||||||
afterIso: string,
|
/** Lower bound, or undefined for "no date bound" (the keep-everything
|
||||||
|
* retention choice). An `after` of undefined must be OMITTED from the
|
||||||
|
* filter, not sent as undefined - Stalwart rejects a malformed filter
|
||||||
|
* rather than treating it as unset. */
|
||||||
|
afterIso: string | undefined,
|
||||||
limit: number,
|
limit: number,
|
||||||
): Promise<string[]> {
|
): Promise<string[]> {
|
||||||
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
const responses = await jmapRequest(session, authHeader, [CAP_CORE, CAP_MAIL], [
|
||||||
['Email/query', {
|
['Email/query', {
|
||||||
accountId,
|
accountId,
|
||||||
filter: { after: afterIso },
|
filter: afterIso ? { after: afterIso } : {},
|
||||||
sort: [{ property: 'receivedAt', isAscending: false }],
|
sort: [{ property: 'receivedAt', isAscending: false }],
|
||||||
limit,
|
limit,
|
||||||
calculateTotal: false,
|
calculateTotal: false,
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
// Recency-intent detection for the retrieval layer.
|
||||||
|
//
|
||||||
|
// Keyword search cannot answer a question about WHEN. bm25 ranks by term
|
||||||
|
// overlap, so "what was the last mail I received" matches documents that
|
||||||
|
// happen to contain the word "last", and "summarise everything from July"
|
||||||
|
// matches documents containing "July" — not documents dated in July. Both were
|
||||||
|
// asked by a real user against a correctly-populated index and both returned
|
||||||
|
// nothing useful, which is what this module exists to fix: it decides when a
|
||||||
|
// question is really a date question, and turns it into a date RANGE the index
|
||||||
|
// can answer with an ordered scan over `occurred_at`.
|
||||||
|
//
|
||||||
|
// Deliberately a heuristic on English/German keywords rather than an LLM call:
|
||||||
|
// it runs on every question, must be instant, and a false positive is cheap
|
||||||
|
// (the recency hits are fused with the keyword hits, not substituted for them).
|
||||||
|
|
||||||
|
// TIMEZONE NOTE: all bounds are built from LOCAL calendar boundaries and then
|
||||||
|
// serialised as UTC instants. That is deliberate — "July" means the user's
|
||||||
|
// July, so a mail received 00:30 local on 1 July belongs to it even though its
|
||||||
|
// stored UTC timestamp reads 30 June. Building the bounds in UTC instead would
|
||||||
|
// silently drop the first/last hours of every named period for anyone not on
|
||||||
|
// UTC.
|
||||||
|
export interface RecencyIntent {
|
||||||
|
/** ISO lower bound, if the question named one. */
|
||||||
|
since?: string;
|
||||||
|
/** ISO upper bound, if the question named a closed period. */
|
||||||
|
until?: string;
|
||||||
|
/** How many documents the recency leg should contribute. */
|
||||||
|
limit: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RECENCY_WORDS = [
|
||||||
|
// English
|
||||||
|
'last', 'latest', 'recent', 'recently', 'newest', 'new', 'today', 'yesterday',
|
||||||
|
'this week', 'this month', 'past week', 'past month', 'so far', 'just now', 'current',
|
||||||
|
// German — the app ships a German UI and users mix languages freely
|
||||||
|
'letzte', 'letzten', 'letzter', 'neueste', 'neuesten', 'neu', 'heute', 'gestern',
|
||||||
|
'diese woche', 'diesen monat', 'kürzlich', 'zuletzt', 'aktuell',
|
||||||
|
];
|
||||||
|
|
||||||
|
const MONTHS: Record<string, number> = {
|
||||||
|
january: 0, february: 1, march: 2, april: 3, may: 4, june: 5,
|
||||||
|
july: 6, august: 7, september: 8, october: 9, november: 10, december: 11,
|
||||||
|
januar: 0, februar: 1, märz: 2, maerz: 2, mai: 4, juni: 5,
|
||||||
|
juli: 6, oktober: 9, dezember: 11,
|
||||||
|
};
|
||||||
|
|
||||||
|
function startOfDay(d: Date): Date {
|
||||||
|
const c = new Date(d);
|
||||||
|
c.setHours(0, 0, 0, 0);
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param now injected so the behaviour is testable and deterministic — the
|
||||||
|
* month-name branch depends on "which year is it" and must not be a coin
|
||||||
|
* flip in a test suite.
|
||||||
|
*/
|
||||||
|
export function detectRecencyIntent(question: string, now: Date = new Date()): RecencyIntent | null {
|
||||||
|
const q = question.toLowerCase();
|
||||||
|
|
||||||
|
// A named month wins over generic recency words: "everything from July" is a
|
||||||
|
// bounded range, which is far more useful than "the newest N".
|
||||||
|
for (const [name, monthIndex] of Object.entries(MONTHS)) {
|
||||||
|
if (!new RegExp(`\\b${name}\\b`).test(q)) continue;
|
||||||
|
// A month later than the current one must mean LAST year — "July" asked in
|
||||||
|
// March means the July that already happened, not one nine months away.
|
||||||
|
const year = monthIndex > now.getMonth() ? now.getFullYear() - 1 : now.getFullYear();
|
||||||
|
const since = new Date(year, monthIndex, 1, 0, 0, 0, 0);
|
||||||
|
const until = new Date(year, monthIndex + 1, 1, 0, 0, 0, 0);
|
||||||
|
return { since: since.toISOString(), until: until.toISOString(), limit: 40 };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (/\btoday\b|\bheute\b/.test(q)) {
|
||||||
|
return { since: startOfDay(now).toISOString(), limit: 25 };
|
||||||
|
}
|
||||||
|
if (/\byesterday\b|\bgestern\b/.test(q)) {
|
||||||
|
const start = startOfDay(new Date(now.getTime() - 86_400_000));
|
||||||
|
return { since: start.toISOString(), until: startOfDay(now).toISOString(), limit: 25 };
|
||||||
|
}
|
||||||
|
if (/this week|past week|diese woche|letzte woche/.test(q)) {
|
||||||
|
return { since: startOfDay(new Date(now.getTime() - 7 * 86_400_000)).toISOString(), limit: 40 };
|
||||||
|
}
|
||||||
|
if (/this month|past month|diesen monat|letzten monat/.test(q)) {
|
||||||
|
return { since: startOfDay(new Date(now.getTime() - 30 * 86_400_000)).toISOString(), limit: 40 };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (RECENCY_WORDS.some((w) => (w.includes(' ') ? q.includes(w) : new RegExp(`\\b${w}\\b`).test(q)))) {
|
||||||
|
// No period named — "the last mail", "what's new". Unbounded top-N.
|
||||||
|
return { limit: 15 };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
+62
-13
@@ -29,16 +29,51 @@ import { withIndexKey } from './key';
|
|||||||
import { getStoreDir } from './paths';
|
import { getStoreDir } from './paths';
|
||||||
import { MailIndex, type ContentType, type IndexDoc } from './store';
|
import { MailIndex, type ContentType, type IndexDoc } from './store';
|
||||||
|
|
||||||
/**
|
|
||||||
* Bounded window. Small on purpose: this is the first cut of a retrieval index,
|
|
||||||
* and a wide window turns "index on every delivery" into a slow request. The
|
|
||||||
* event-driven path indexes single objects, so the window only bounds catch-up.
|
|
||||||
*/
|
|
||||||
export const INDEX_WINDOW_DAYS = 30;
|
|
||||||
/** Calendar looks forward as well as back - upcoming events are the useful ones. */
|
/** Calendar looks forward as well as back - upcoming events are the useful ones. */
|
||||||
export const CALENDAR_FORWARD_DAYS = 180;
|
export const CALENDAR_FORWARD_DAYS = 180;
|
||||||
/** Per-type ceiling for one catch-up pass. */
|
/** Per-type ceiling for one catch-up pass, per 30 days of window. */
|
||||||
export const CATCHUP_MAX_PER_TYPE = 500;
|
export const CATCHUP_MAX_PER_TYPE = 500;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How much mail history the local index keeps. User-selectable
|
||||||
|
* (Settings -> About & Data); `null` means keep everything and never prune.
|
||||||
|
*
|
||||||
|
* This is NOT just a fetch bound - catch-up also PRUNES mail older than it.
|
||||||
|
* The original hardcoded 30 days therefore made a question like "summarise
|
||||||
|
* everything from July" unanswerable in August: the rows had been deliberately
|
||||||
|
* deleted, while the UI said only that nothing matched. A real user hit exactly
|
||||||
|
* that, which is why this is a setting with a year-long default rather than a
|
||||||
|
* constant tuned for a first cut.
|
||||||
|
*/
|
||||||
|
export const RETENTION_CHOICES = [30, 90, 365, null] as const;
|
||||||
|
export type RetentionWindowDays = (typeof RETENTION_CHOICES)[number];
|
||||||
|
export const DEFAULT_RETENTION_WINDOW_DAYS: RetentionWindowDays = 365;
|
||||||
|
|
||||||
|
/** Hard ceiling regardless of window - one pass must still terminate. */
|
||||||
|
export const CATCHUP_MAX_PER_TYPE_UNLIMITED = 20_000;
|
||||||
|
|
||||||
|
export function normalizeWindowDays(raw: unknown): RetentionWindowDays {
|
||||||
|
if (raw === null) return null;
|
||||||
|
if (typeof raw !== 'number' || !Number.isFinite(raw)) return DEFAULT_RETENTION_WINDOW_DAYS;
|
||||||
|
// Anything off the list falls back to the default rather than being honoured
|
||||||
|
// verbatim - this value drives DELETION, so a typo'd 0 must never silently
|
||||||
|
// wipe the index.
|
||||||
|
const allowed: readonly number[] = RETENTION_CHOICES.filter((c) => c !== null);
|
||||||
|
return allowed.includes(raw) ? (raw as RetentionWindowDays) : DEFAULT_RETENTION_WINDOW_DAYS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scales the per-pass ceiling with the window: 500 is right for a month and
|
||||||
|
* nonsense for "everything", where having the history IS the point. */
|
||||||
|
export function catchUpCapFor(windowDays: RetentionWindowDays): number {
|
||||||
|
if (windowDays === null) return CATCHUP_MAX_PER_TYPE_UNLIMITED;
|
||||||
|
return Math.min(CATCHUP_MAX_PER_TYPE_UNLIMITED, Math.round((windowDays / 30) * CATCHUP_MAX_PER_TYPE));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Lower bound for a date-filtered query, or undefined when unlimited. */
|
||||||
|
export function windowStartIso(windowDays: RetentionWindowDays): string | undefined {
|
||||||
|
return windowDays === null ? undefined : isoDaysFromNow(-windowDays);
|
||||||
|
}
|
||||||
|
|
||||||
/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
|
/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
|
||||||
export const MAX_IDS_PER_CALL = 200;
|
export const MAX_IDS_PER_CALL = 200;
|
||||||
/** Cap on body bytes requested per message from the server. */
|
/** Cap on body bytes requested per message from the server. */
|
||||||
@@ -169,6 +204,7 @@ interface FetchArgs {
|
|||||||
authHeader: string;
|
authHeader: string;
|
||||||
jmapAccountId: string;
|
jmapAccountId: string;
|
||||||
ids: readonly string[] | null;
|
ids: readonly string[] | null;
|
||||||
|
windowDays: RetentionWindowDays;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface FetchResult {
|
interface FetchResult {
|
||||||
@@ -184,13 +220,14 @@ interface FetchResult {
|
|||||||
|
|
||||||
/** Fetches and flattens one content type. `ids === null` means "the recent window". */
|
/** Fetches and flattens one content type. `ids === null` means "the recent window". */
|
||||||
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<FetchResult> {
|
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<FetchResult> {
|
||||||
const { session, authHeader, jmapAccountId, ids } = args;
|
const { session, authHeader, jmapAccountId, ids, windowDays } = args;
|
||||||
|
const cap = catchUpCapFor(windowDays);
|
||||||
|
|
||||||
switch (contentType) {
|
switch (contentType) {
|
||||||
case 'mail': {
|
case 'mail': {
|
||||||
const targetIds = ids ?? await queryRecentEmailIds(
|
const targetIds = ids ?? await queryRecentEmailIds(
|
||||||
session, authHeader, jmapAccountId,
|
session, authHeader, jmapAccountId,
|
||||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE,
|
windowStartIso(windowDays), cap,
|
||||||
);
|
);
|
||||||
const docs: IndexDoc[] = [];
|
const docs: IndexDoc[] = [];
|
||||||
// Chunked because bodies are big: one Email/get for 500 messages with
|
// Chunked because bodies are big: one Email/get for 500 messages with
|
||||||
@@ -206,8 +243,11 @@ async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<Fet
|
|||||||
case 'calendar': {
|
case 'calendar': {
|
||||||
const targetIds = ids ?? await queryCalendarEventIds(
|
const targetIds = ids ?? await queryCalendarEventIds(
|
||||||
session, authHeader, jmapAccountId,
|
session, authHeader, jmapAccountId,
|
||||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
// Calendar keeps its own bounded look-back even under "keep
|
||||||
CATCHUP_MAX_PER_TYPE,
|
// everything": events are small but a decade of them is noise in a
|
||||||
|
// mail assistant's context, and the forward window is the useful half.
|
||||||
|
windowStartIso(windowDays) ?? isoDaysFromNow(-365), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||||
|
cap,
|
||||||
);
|
);
|
||||||
const docs: IndexDoc[] = [];
|
const docs: IndexDoc[] = [];
|
||||||
for (let i = 0; i < targetIds.length; i += 50) {
|
for (let i = 0; i < targetIds.length; i += 50) {
|
||||||
@@ -260,6 +300,10 @@ export interface IndexRequest {
|
|||||||
removed?: Partial<Record<ContentType, readonly string[]>>;
|
removed?: Partial<Record<ContentType, readonly string[]>>;
|
||||||
/** Drop documents outside the retention window after writing. */
|
/** Drop documents outside the retention window after writing. */
|
||||||
prune?: boolean;
|
prune?: boolean;
|
||||||
|
/** Retention window in days, or null to keep everything. Bounds BOTH the
|
||||||
|
* catch-up fetch and the prune, so the two can never disagree and delete
|
||||||
|
* what was just written. Defaults to DEFAULT_RETENTION_WINDOW_DAYS. */
|
||||||
|
windowDays?: RetentionWindowDays;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -274,6 +318,7 @@ export async function runIndex(
|
|||||||
req: IndexRequest,
|
req: IndexRequest,
|
||||||
): Promise<IndexResult> {
|
): Promise<IndexResult> {
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
|
const windowDays = normalizeWindowDays(req.windowDays === undefined ? DEFAULT_RETENTION_WINDOW_DAYS : req.windowDays);
|
||||||
const storeDir = getStoreDir();
|
const storeDir = getStoreDir();
|
||||||
if (!storeDir) {
|
if (!storeDir) {
|
||||||
throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
|
throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
|
||||||
@@ -325,14 +370,18 @@ export async function runIndex(
|
|||||||
: null;
|
: null;
|
||||||
|
|
||||||
const { docs, queriedCount } = await fetchDocs(contentType, {
|
const { docs, queriedCount } = await fetchDocs(contentType, {
|
||||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids,
|
session, authHeader: indexSession.authHeader, jmapAccountId, ids, windowDays,
|
||||||
});
|
});
|
||||||
written[contentType] = index.upsert(docs);
|
written[contentType] = index.upsert(docs);
|
||||||
|
|
||||||
if (req.prune && contentType === 'mail') {
|
if (req.prune && contentType === 'mail') {
|
||||||
// Only mail prunes by date: calendar's window looks forward,
|
// Only mail prunes by date: calendar's window looks forward,
|
||||||
// contacts have no date, and file rows are metadata-sized.
|
// contacts have no date, and file rows are metadata-sized.
|
||||||
index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS));
|
// A null window means keep everything - pruning is SKIPPED, not
|
||||||
|
// run with some fallback bound, or the setting would silently
|
||||||
|
// delete the history the user just asked to retain.
|
||||||
|
const cutoff = windowStartIso(windowDays);
|
||||||
|
if (cutoff) index.pruneOlderThan(jmapAccountId, 'mail', cutoff);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Contact/file DELETES: a JMAP `destroyed` only ever reaches this
|
// Contact/file DELETES: a JMAP `destroyed` only ever reaches this
|
||||||
|
|||||||
@@ -374,6 +374,69 @@ export class MailIndex {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Newest documents by date, ignoring keyword relevance entirely.
|
||||||
|
*
|
||||||
|
* The retrieval leg for RECENCY questions — "what was the last mail", "who
|
||||||
|
* wrote most recently", "everything from July". Full-text search cannot
|
||||||
|
* answer those even with a perfect index: bm25 ranks by term overlap and has
|
||||||
|
* no notion of "latest", so "the last mail" matches documents containing the
|
||||||
|
* word "last". Two real questions failed exactly that way before this
|
||||||
|
* existed. `doc(jmap_account_id, content_type, occurred_at DESC)` is already
|
||||||
|
* indexed, so this is an ordered range scan, not a table sweep.
|
||||||
|
*
|
||||||
|
* `since`/`until` are ISO strings, both optional — a month-name question
|
||||||
|
* becomes a bounded range, a bare "latest" becomes an unbounded top-N.
|
||||||
|
*/
|
||||||
|
recent(opts: {
|
||||||
|
types?: readonly ContentType[];
|
||||||
|
limit?: number;
|
||||||
|
since?: string;
|
||||||
|
until?: string;
|
||||||
|
snippetChars?: number;
|
||||||
|
}): SearchHit[] {
|
||||||
|
const limit = Math.min(Math.max(opts.limit ?? 10, 1), 200);
|
||||||
|
const snippetChars = Math.min(Math.max(opts.snippetChars ?? 400, 80), 2000);
|
||||||
|
const types = opts.types && opts.types.length > 0 ? opts.types : null;
|
||||||
|
|
||||||
|
const where: string[] = ['d.occurred_at IS NOT NULL'];
|
||||||
|
const params: Array<string | number> = [];
|
||||||
|
if (types) {
|
||||||
|
where.push(`d.content_type IN (${types.map(() => '?').join(',')})`);
|
||||||
|
params.push(...types);
|
||||||
|
}
|
||||||
|
if (opts.since) { where.push('d.occurred_at >= ?'); params.push(opts.since); }
|
||||||
|
if (opts.until) { where.push('d.occurred_at <= ?'); params.push(opts.until); }
|
||||||
|
|
||||||
|
const rows = this.db
|
||||||
|
.prepare(`
|
||||||
|
SELECT d.content_type, d.id, d.jmap_account_id, d.title, d.people,
|
||||||
|
d.occurred_at, d.metadata_json,
|
||||||
|
substr(f.body, 1, ${snippetChars}) AS snip
|
||||||
|
FROM doc d
|
||||||
|
LEFT JOIN doc_fts f ON f.rowid = d.rowid
|
||||||
|
WHERE ${where.join(' AND ')}
|
||||||
|
ORDER BY d.occurred_at DESC
|
||||||
|
LIMIT ?
|
||||||
|
`)
|
||||||
|
.all([...params, limit]);
|
||||||
|
|
||||||
|
return rows.map((r) => ({
|
||||||
|
contentType: String(r.content_type) as ContentType,
|
||||||
|
id: String(r.id),
|
||||||
|
jmapAccountId: String(r.jmap_account_id),
|
||||||
|
title: String(r.title ?? ''),
|
||||||
|
people: String(r.people ?? ''),
|
||||||
|
occurredAt: r.occurred_at === null || r.occurred_at === undefined ? null : String(r.occurred_at),
|
||||||
|
metadata: safeParseObject(r.metadata_json),
|
||||||
|
// No bm25 score here: these are ordered by time, not relevance, and
|
||||||
|
// faking a relevance number would let the fusion step rank them as if
|
||||||
|
// they had been scored.
|
||||||
|
score: 0,
|
||||||
|
snippet: String(r.snip ?? ''),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
/** Per-type counts and freshness, for the Settings UI and for debugging. */
|
/** Per-type counts and freshness, for the Settings UI and for debugging. */
|
||||||
stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> {
|
stats(): Array<{ contentType: string; count: number; newest: string | null; indexedAt: number | null }> {
|
||||||
return this.db
|
return this.db
|
||||||
|
|||||||
Reference in New Issue
Block a user