feat(mail-index): 1-year retention by default + a recency retrieval leg
The two things that made real questions fail against a correctly-populated index, both fixed at the root. RETENTION (A1). `INDEX_WINDOW_DAYS = 30` was not merely a fetch bound — catch-up also PRUNED mail older than it, so "summarise everything from July" was unanswerable in August because the rows had been deleted, while the UI said only that nothing matched. Now a user-visible setting (Settings → About & Data): 30 days / 3 months / 1 year / everything, defaulting to 1 YEAR per the product owner. The window bounds the fetch AND the prune from one value so the two can never disagree and delete what was just written; "everything" skips pruning entirely rather than falling back to some default bound. The per-pass ceiling scales with the window (500/30d, hard cap 20k) because 500 messages is right for a month and nonsense for "everything". Email/query now omits the `after` filter entirely when unbounded — Stalwart rejects a malformed filter rather than treating `undefined` as unset. RECENCY (A2). Keyword search structurally cannot answer a question about WHEN: bm25 ranks by term overlap, so "who sent the last email" matches documents containing the word "last", and "all mails in July" matches documents containing "July" — not documents dated in July. Both were asked by a real user and both failed. New lib/mail-index/recency.ts detects time intent (English + German, since the UI ships German) and turns it into a date RANGE; new MailIndex.recent() answers it with an ordered scan over the already-indexed `occurred_at`. The route ADDS these hits to the keyword hits rather than replacing them — "what did the last mail from Anna say" is both kinds of question at once. Timezone subtlety worth knowing: bounds are built from LOCAL calendar boundaries and serialised as UTC instants, so "July" covers the user's July. A mail at 00:30 local on 1 July belongs to it even though its stored UTC timestamp reads 30 June. My first test asserted the ISO string prefix, which would have enshrined the opposite and passed only in UTC — the tests now assert the local-time property instead. SCOPE, stated by the product owner and now enforced structurally: the assistant only ever sees the mailbox the user is signed in to. Both retrieval legs resolve the active account (local leg by cookie slot, server leg by the session's own JMAP account); there is deliberately no fan-out across connected or shared mailboxes, and adding one would be a policy change, not a feature. Gate: tsc clean, eslint clean, 2520/2520 tests (8 new for recency intent), build clean.
This commit is contained in:
Bernd Rodler
+62
-13
@@ -29,16 +29,51 @@ import { withIndexKey } from './key';
|
||||
import { getStoreDir } from './paths';
|
||||
import { MailIndex, type ContentType, type IndexDoc } from './store';
|
||||
|
||||
/**
|
||||
* Bounded window. Small on purpose: this is the first cut of a retrieval index,
|
||||
* and a wide window turns "index on every delivery" into a slow request. The
|
||||
* event-driven path indexes single objects, so the window only bounds catch-up.
|
||||
*/
|
||||
export const INDEX_WINDOW_DAYS = 30;
|
||||
/** Calendar looks forward as well as back - upcoming events are the useful ones. */
|
||||
export const CALENDAR_FORWARD_DAYS = 180;
|
||||
/** Per-type ceiling for one catch-up pass. */
|
||||
/** Per-type ceiling for one catch-up pass, per 30 days of window. */
|
||||
export const CATCHUP_MAX_PER_TYPE = 500;
|
||||
|
||||
/**
|
||||
* How much mail history the local index keeps. User-selectable
|
||||
* (Settings -> About & Data); `null` means keep everything and never prune.
|
||||
*
|
||||
* This is NOT just a fetch bound - catch-up also PRUNES mail older than it.
|
||||
* The original hardcoded 30 days therefore made a question like "summarise
|
||||
* everything from July" unanswerable in August: the rows had been deliberately
|
||||
* deleted, while the UI said only that nothing matched. A real user hit exactly
|
||||
* that, which is why this is a setting with a year-long default rather than a
|
||||
* constant tuned for a first cut.
|
||||
*/
|
||||
export const RETENTION_CHOICES = [30, 90, 365, null] as const;
|
||||
export type RetentionWindowDays = (typeof RETENTION_CHOICES)[number];
|
||||
export const DEFAULT_RETENTION_WINDOW_DAYS: RetentionWindowDays = 365;
|
||||
|
||||
/** Hard ceiling regardless of window - one pass must still terminate. */
|
||||
export const CATCHUP_MAX_PER_TYPE_UNLIMITED = 20_000;
|
||||
|
||||
export function normalizeWindowDays(raw: unknown): RetentionWindowDays {
|
||||
if (raw === null) return null;
|
||||
if (typeof raw !== 'number' || !Number.isFinite(raw)) return DEFAULT_RETENTION_WINDOW_DAYS;
|
||||
// Anything off the list falls back to the default rather than being honoured
|
||||
// verbatim - this value drives DELETION, so a typo'd 0 must never silently
|
||||
// wipe the index.
|
||||
const allowed: readonly number[] = RETENTION_CHOICES.filter((c) => c !== null);
|
||||
return allowed.includes(raw) ? (raw as RetentionWindowDays) : DEFAULT_RETENTION_WINDOW_DAYS;
|
||||
}
|
||||
|
||||
/** Scales the per-pass ceiling with the window: 500 is right for a month and
|
||||
* nonsense for "everything", where having the history IS the point. */
|
||||
export function catchUpCapFor(windowDays: RetentionWindowDays): number {
|
||||
if (windowDays === null) return CATCHUP_MAX_PER_TYPE_UNLIMITED;
|
||||
return Math.min(CATCHUP_MAX_PER_TYPE_UNLIMITED, Math.round((windowDays / 30) * CATCHUP_MAX_PER_TYPE));
|
||||
}
|
||||
|
||||
/** Lower bound for a date-filtered query, or undefined when unlimited. */
|
||||
export function windowStartIso(windowDays: RetentionWindowDays): string | undefined {
|
||||
return windowDays === null ? undefined : isoDaysFromNow(-windowDays);
|
||||
}
|
||||
|
||||
/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
|
||||
export const MAX_IDS_PER_CALL = 200;
|
||||
/** Cap on body bytes requested per message from the server. */
|
||||
@@ -169,6 +204,7 @@ interface FetchArgs {
|
||||
authHeader: string;
|
||||
jmapAccountId: string;
|
||||
ids: readonly string[] | null;
|
||||
windowDays: RetentionWindowDays;
|
||||
}
|
||||
|
||||
interface FetchResult {
|
||||
@@ -184,13 +220,14 @@ interface FetchResult {
|
||||
|
||||
/** Fetches and flattens one content type. `ids === null` means "the recent window". */
|
||||
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<FetchResult> {
|
||||
const { session, authHeader, jmapAccountId, ids } = args;
|
||||
const { session, authHeader, jmapAccountId, ids, windowDays } = args;
|
||||
const cap = catchUpCapFor(windowDays);
|
||||
|
||||
switch (contentType) {
|
||||
case 'mail': {
|
||||
const targetIds = ids ?? await queryRecentEmailIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), CATCHUP_MAX_PER_TYPE,
|
||||
windowStartIso(windowDays), cap,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
// Chunked because bodies are big: one Email/get for 500 messages with
|
||||
@@ -206,8 +243,11 @@ async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<Fet
|
||||
case 'calendar': {
|
||||
const targetIds = ids ?? await queryCalendarEventIds(
|
||||
session, authHeader, jmapAccountId,
|
||||
isoDaysFromNow(-INDEX_WINDOW_DAYS), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||
CATCHUP_MAX_PER_TYPE,
|
||||
// Calendar keeps its own bounded look-back even under "keep
|
||||
// everything": events are small but a decade of them is noise in a
|
||||
// mail assistant's context, and the forward window is the useful half.
|
||||
windowStartIso(windowDays) ?? isoDaysFromNow(-365), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
|
||||
cap,
|
||||
);
|
||||
const docs: IndexDoc[] = [];
|
||||
for (let i = 0; i < targetIds.length; i += 50) {
|
||||
@@ -260,6 +300,10 @@ export interface IndexRequest {
|
||||
removed?: Partial<Record<ContentType, readonly string[]>>;
|
||||
/** Drop documents outside the retention window after writing. */
|
||||
prune?: boolean;
|
||||
/** Retention window in days, or null to keep everything. Bounds BOTH the
|
||||
* catch-up fetch and the prune, so the two can never disagree and delete
|
||||
* what was just written. Defaults to DEFAULT_RETENTION_WINDOW_DAYS. */
|
||||
windowDays?: RetentionWindowDays;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -274,6 +318,7 @@ export async function runIndex(
|
||||
req: IndexRequest,
|
||||
): Promise<IndexResult> {
|
||||
const started = Date.now();
|
||||
const windowDays = normalizeWindowDays(req.windowDays === undefined ? DEFAULT_RETENTION_WINDOW_DAYS : req.windowDays);
|
||||
const storeDir = getStoreDir();
|
||||
if (!storeDir) {
|
||||
throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
|
||||
@@ -325,14 +370,18 @@ export async function runIndex(
|
||||
: null;
|
||||
|
||||
const { docs, queriedCount } = await fetchDocs(contentType, {
|
||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids,
|
||||
session, authHeader: indexSession.authHeader, jmapAccountId, ids, windowDays,
|
||||
});
|
||||
written[contentType] = index.upsert(docs);
|
||||
|
||||
if (req.prune && contentType === 'mail') {
|
||||
// Only mail prunes by date: calendar's window looks forward,
|
||||
// contacts have no date, and file rows are metadata-sized.
|
||||
index.pruneOlderThan(jmapAccountId, 'mail', isoDaysFromNow(-INDEX_WINDOW_DAYS));
|
||||
// A null window means keep everything - pruning is SKIPPED, not
|
||||
// run with some fallback bound, or the setting would silently
|
||||
// delete the history the user just asked to retain.
|
||||
const cutoff = windowStartIso(windowDays);
|
||||
if (cutoff) index.pruneOlderThan(jmapAccountId, 'mail', cutoff);
|
||||
}
|
||||
|
||||
// Contact/file DELETES: a JMAP `destroyed` only ever reaches this
|
||||
|
||||
Reference in New Issue
Block a user