SRCmail/lib/mail-index/reindex.ts

// The index jobs.
//
// TWO SHAPES, both plain request-scoped work - there is no background worker,
// no cursor, no retry ladder and no resident credential anywhere:
//
//  1. `indexDocuments()` - the PRIMARY path. The renderer's live JMAP push
//     connection sees a StateChange, and calls the route with the ids that
//     changed (or with no ids, meaning "refetch what's recent for this type").
//     One or a handful of objects, fetched and upserted.
//  2. `catchUpAll()` - the FALLBACK. On app launch, backfill a bounded recent
//     window for every supported type, because anything that changed while the
//     app was closed produced no push event.
//
// Staleness between refreshes is acceptable by design: this is a search index
// for a retrieval/AI feature, not a mail replica.

import type { NextRequest } from 'next/server';
import { generateAccountId } from '@/lib/account-utils';
import { getStalwartCredentials } from '@/lib/stalwart/credentials';
import { logger } from '@/lib/logger';
import {
  accountHasCapability, accountIdFor, buildFilePaths, CAP_CALENDARS, CAP_CONTACTS,
  CAP_FILENODE, CAP_MAIL, fetchJmapSession, getCalendarEventsForIndex, getContactsForIndex,
  getEmailsForIndex, getFilesForIndex, hasCapability, JmapIndexError, queryCalendarEventIds,
  queryContactIds, queryFileIds, queryRecentEmailIds, type JmapSessionInfo,
} from './jmap';
import { extractCalendarEvent, extractContact, extractFile, extractMail } from './extract';
import { withIndexKey } from './key';
import { getStoreDir } from './paths';
import { MailIndex, type ContentType, type IndexDoc } from './store';

/** Calendar looks forward as well as back - upcoming events are the useful ones. */
export const CALENDAR_FORWARD_DAYS = 180;
/** Per-type ceiling for one catch-up pass, per 30 days of window. */
export const CATCHUP_MAX_PER_TYPE = 500;

/**
 * How much mail history the local index keeps. User-selectable
 * (Settings -> About & Data); `null` means keep everything and never prune.
 *
 * This is NOT just a fetch bound - catch-up also PRUNES mail older than it.
 * The original hardcoded 30 days therefore made a question like "summarise
 * everything from July" unanswerable in August: the rows had been deliberately
 * deleted, while the UI said only that nothing matched. A real user hit exactly
 * that, which is why this is a setting with a year-long default rather than a
 * constant tuned for a first cut.
 */
export const RETENTION_CHOICES = [30, 90, 365, null] as const;
export type RetentionWindowDays = (typeof RETENTION_CHOICES)[number];
export const DEFAULT_RETENTION_WINDOW_DAYS: RetentionWindowDays = 365;

/** Hard ceiling regardless of window - one pass must still terminate. */
export const CATCHUP_MAX_PER_TYPE_UNLIMITED = 20_000;

export function normalizeWindowDays(raw: unknown): RetentionWindowDays {
  if (raw === null) return null;
  if (typeof raw !== 'number' || !Number.isFinite(raw)) return DEFAULT_RETENTION_WINDOW_DAYS;
  // Anything off the list falls back to the default rather than being honoured
  // verbatim - this value drives DELETION, so a typo'd 0 must never silently
  // wipe the index.
  const allowed: readonly number[] = RETENTION_CHOICES.filter((c) => c !== null);
  return allowed.includes(raw) ? (raw as RetentionWindowDays) : DEFAULT_RETENTION_WINDOW_DAYS;
}

/** Scales the per-pass ceiling with the window: 500 is right for a month and
 *  nonsense for "everything", where having the history IS the point. */
export function catchUpCapFor(windowDays: RetentionWindowDays): number {
  if (windowDays === null) return CATCHUP_MAX_PER_TYPE_UNLIMITED;
  return Math.min(CATCHUP_MAX_PER_TYPE_UNLIMITED, Math.round((windowDays / 30) * CATCHUP_MAX_PER_TYPE));
}

/** Lower bound for a date-filtered query, or undefined when unlimited. */
export function windowStartIso(windowDays: RetentionWindowDays): string | undefined {
  return windowDays === null ? undefined : isoDaysFromNow(-windowDays);
}

/** Ids accepted in one event-driven call. A push reports a handful, not thousands. */
export const MAX_IDS_PER_CALL = 200;
/** Cap on body bytes requested per message from the server. */
export const MAX_BODY_VALUE_BYTES = 256_000;
/** Contacts and files have no useful date filter, so they are simply capped. */
export const CONTACTS_MAX = 2_000;
export const FILES_MAX = 2_000;

export interface IndexSession {
  serverUrl: string;
  authHeader: string;
  username: string;
  slot: number;
  /** `username@host` - the durable per-account key. NEVER the cookie slot. */
  accountId: string;
}

export class IndexSessionError extends Error {
  status: number;
  constructor(message: string, status: number) {
    super(message);
    this.name = 'IndexSessionError';
    this.status = status;
  }
}

/**
 * Resolves the calling request to an account and a usable Authorization header.
 *
 * Uses the SAME per-slot encrypted `jmap_stalwart_ctx` cookie that
 * `/api/settings`, `/api/push/preview` and `/api/plugin-approval-status`
 * already read (`lib/stalwart/credentials.ts`). That cookie is written by
 * `/api/auth/stalwart-context`, which the renderer syncs on every login,
 * session restore, SSO callback, account switch and token refresh
 * (`stores/auth-store.ts`, 10 call sites), and it carries a ready-made header
 * for BOTH basic and bearer accounts.
 *
 * Why this matters beyond convenience: it means the indexer never touches the
 * OAuth refresh-token cookie. A server-side refresh would rotate the token into
 * a response nobody reads while the browser kept the superseded one, and the
 * next real refresh would then fail and log the user out. Reading an
 * already-minted header cannot cause that.
 */
export async function resolveIndexSession(request: NextRequest): Promise<IndexSession> {
  const credentials = await getStalwartCredentials(request);
  if (!credentials) {
    throw new IndexSessionError('No JMAP auth context for this account; sign in again.', 401);
  }
  const accountId = generateAccountId(credentials.username, credentials.serverUrl);
  return { ...credentials, accountId };
}

export interface IndexResult {
  accountId: string;
  /** Per-type counts of documents written. */
  written: Partial<Record<ContentType, number>>;
  /** Types the server (or this account) doesn't support, so nothing was attempted. */
  skipped: ContentType[];
  /** Non-fatal per-type failures. One broken type must not fail the whole call. */
  errors: Array<{ contentType: ContentType; message: string }>;
  durationMs: number;
}

function isoDaysFromNow(days: number): string {
  return new Date(Date.now() + days * 24 * 60 * 60 * 1000).toISOString();
}

/**
 * Which locally-indexed ids are stray after an uncapped (contact/file)
 * catch-up fetch, and therefore safe to remove as deleted.
 *
 * Only safe when `queriedCount < cap`: a query that hit the cap was
 * truncated - "the rest weren't asked for", not "the rest are gone" - and
 * treating a truncated page as the whole world would delete objects that are
 * still live. Exported for unit testing; the database-touching caller is not.
 */
export function strayIdsAfterCatchUp(
  existingIds: ReadonlySet<string>,
  fetchedIds: readonly string[],
  queriedCount: number,
  cap: number,
): string[] {
  if (queriedCount >= cap) return [];
  const fetched = new Set(fetchedIds);
  return [...existingIds].filter((id) => !fetched.has(id));
}

/**
 * Which types this session can actually index. Calendar/contacts are session
 * capabilities; files is a PER-ACCOUNT capability (a server can advertise
 * filenode while a specific account has it revoked - #563).
 */
export function supportedTypes(session: JmapSessionInfo): {
  supported: ContentType[];
  skipped: ContentType[];
  accountIds: Partial<Record<ContentType, string>>;
} {
  const supported: ContentType[] = [];
  const skipped: ContentType[] = [];
  const accountIds: Partial<Record<ContentType, string>> = {};

  const mailAccount = accountIdFor(session, CAP_MAIL);
  if (mailAccount) { supported.push('mail'); accountIds.mail = mailAccount; }
  else skipped.push('mail');

  const calAccount = accountIdFor(session, CAP_CALENDARS);
  if (calAccount && hasCapability(session, CAP_CALENDARS)) {
    supported.push('calendar'); accountIds.calendar = calAccount;
  } else skipped.push('calendar');

  const contactAccount = accountIdFor(session, CAP_CONTACTS);
  if (contactAccount && hasCapability(session, CAP_CONTACTS)) {
    supported.push('contact'); accountIds.contact = contactAccount;
  } else skipped.push('contact');

  // Files fall back to the mail account id: Stalwart exposes FileNode on the
  // same account and does not always list a primaryAccounts entry for it.
  const fileAccount = accountIdFor(session, CAP_FILENODE) ?? mailAccount;
  if (fileAccount && accountHasCapability(session, fileAccount, CAP_FILENODE)) {
    supported.push('file'); accountIds.file = fileAccount;
  } else skipped.push('file');

  return { supported, skipped, accountIds };
}

interface FetchArgs {
  session: JmapSessionInfo;
  authHeader: string;
  jmapAccountId: string;
  ids: readonly string[] | null;
  windowDays: RetentionWindowDays;
}

interface FetchResult {
  docs: IndexDoc[];
  /**
   * How many ids the type's OWN query returned, before any `Foo/get`
   * chunking. Only set when `ids === null` (a catch-up fetch); used to tell a
   * complete uncapped fetch apart from one truncated at its cap - see
   * `strayIdsAfterCatchUp`, the only consumer.
   */
  queriedCount?: number;
}

/** Fetches and flattens one content type. `ids === null` means "the recent window". */
async function fetchDocs(contentType: ContentType, args: FetchArgs): Promise<FetchResult> {
  const { session, authHeader, jmapAccountId, ids, windowDays } = args;
  const cap = catchUpCapFor(windowDays);

  switch (contentType) {
    case 'mail': {
      const targetIds = ids ?? await queryRecentEmailIds(
        session, authHeader, jmapAccountId,
        windowStartIso(windowDays), cap,
      );
      const docs: IndexDoc[] = [];
      // Chunked because bodies are big: one Email/get for 500 messages with
      // full bodies would be an enormous response.
      for (let i = 0; i < targetIds.length; i += 25) {
        const emails = await getEmailsForIndex(
          session, authHeader, jmapAccountId, targetIds.slice(i, i + 25), MAX_BODY_VALUE_BYTES,
        );
        for (const email of emails) docs.push(extractMail(jmapAccountId, email));
      }
      return { docs, queriedCount: ids ? undefined : targetIds.length };
    }
    case 'calendar': {
      const targetIds = ids ?? await queryCalendarEventIds(
        session, authHeader, jmapAccountId,
        // Calendar keeps its own bounded look-back even under "keep
        // everything": events are small but a decade of them is noise in a
        // mail assistant's context, and the forward window is the useful half.
        windowStartIso(windowDays) ?? isoDaysFromNow(-365), isoDaysFromNow(CALENDAR_FORWARD_DAYS),
        cap,
      );
      const docs: IndexDoc[] = [];
      for (let i = 0; i < targetIds.length; i += 50) {
        const events = await getCalendarEventsForIndex(
          session, authHeader, jmapAccountId, targetIds.slice(i, i + 50),
        );
        for (const event of events) docs.push(extractCalendarEvent(jmapAccountId, event));
      }
      return { docs, queriedCount: ids ? undefined : targetIds.length };
    }
    case 'contact': {
      const targetIds = ids ?? await queryContactIds(session, authHeader, jmapAccountId, CONTACTS_MAX);
      const docs: IndexDoc[] = [];
      for (let i = 0; i < targetIds.length; i += 100) {
        const cards = await getContactsForIndex(
          session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
        );
        for (const card of cards) docs.push(extractContact(jmapAccountId, card));
      }
      return { docs, queriedCount: ids ? undefined : targetIds.length };
    }
    case 'file': {
      const targetIds = ids ?? await queryFileIds(session, authHeader, jmapAccountId, FILES_MAX);
      const nodes = [];
      for (let i = 0; i < targetIds.length; i += 100) {
        nodes.push(...await getFilesForIndex(
          session, authHeader, jmapAccountId, targetIds.slice(i, i + 100),
        ));
      }
      // Paths need the whole set in hand, so this one can't stream per chunk.
      const paths = buildFilePaths(nodes);
      const docs = nodes
        // Directories are indexed too: "what's in the Invoices folder" is a
        // real query, and a folder row is a few bytes.
        .map((node) => extractFile(jmapAccountId, node, { path: paths.get(node.id) }));
      return { docs, queriedCount: ids ? undefined : targetIds.length };
    }
  }
}

export interface IndexRequest {
  /** Types to touch. Empty means every supported type. */
  types?: readonly ContentType[];
  /**
   * Per-type ids to index. Omitted/empty for a type means "refetch that type's
   * recent window" (the catch-up shape).
   */
  ids?: Partial<Record<ContentType, readonly string[]>>;
  /** Per-type ids to REMOVE (a JMAP `destroyed`). */
  removed?: Partial<Record<ContentType, readonly string[]>>;
  /** Drop documents outside the retention window after writing. */
  prune?: boolean;
  /** Retention window in days, or null to keep everything. Bounds BOTH the
   *  catch-up fetch and the prune, so the two can never disagree and delete
   *  what was just written. Defaults to DEFAULT_RETENTION_WINDOW_DAYS. */
  windowDays?: RetentionWindowDays;
}

/**
 * Runs one index pass. Opens the encrypted store, fetches, upserts, closes.
 *
 * The key is fetched from the main process for the duration of this call only
 * (`withIndexKey`) and zeroed afterwards - there is no cached handle and no
 * resident key.
 */
export async function runIndex(
  indexSession: IndexSession,
  req: IndexRequest,
): Promise<IndexResult> {
  const started = Date.now();
  const windowDays = normalizeWindowDays(req.windowDays === undefined ? DEFAULT_RETENTION_WINDOW_DAYS : req.windowDays);
  const storeDir = getStoreDir();
  if (!storeDir) {
    throw new IndexSessionError('The local index is not enabled in this deployment.', 404);
  }

  const session = await fetchJmapSession(indexSession.serverUrl, indexSession.authHeader);

  // Identity cross-check. `generateAccountId` used the username from the auth
  // context cookie; the server may canonicalise a short login (`linus`) to a
  // full address (`linus@example.com`) - which is exactly why AccountEntry
  // carries `serverIdentifiers`. Accept either form, reject anything else
  // rather than writing one account's mail into another's file.
  if (session.username) {
    const serverAccountId = generateAccountId(session.username, indexSession.serverUrl);
    if (serverAccountId !== indexSession.accountId) {
      const shortMatches = session.username.split('@')[0] === indexSession.username.split('@')[0];
      if (!shortMatches) {
        throw new IndexSessionError(
          'The JMAP session belongs to a different account than the request cookie.',
          409,
        );
      }
    }
  }

  const { supported, skipped, accountIds } = supportedTypes(session);
  const requested = req.types && req.types.length > 0 ? req.types : supported;
  const types = requested.filter((t) => supported.includes(t));
  const notAttempted = [...new Set([...skipped, ...requested.filter((t) => !supported.includes(t))])];

  const written: Partial<Record<ContentType, number>> = {};
  const errors: IndexResult['errors'] = [];

  await withIndexKey(indexSession.accountId, async (key) => {
    const index = MailIndex.open({ storeDir, accountId: indexSession.accountId, key });
    try {
      for (const contentType of types) {
        const jmapAccountId = accountIds[contentType];
        if (!jmapAccountId) continue;
        try {
          const removed = req.removed?.[contentType];
          if (removed && removed.length > 0) {
            index.remove(jmapAccountId, contentType, removed.slice(0, MAX_IDS_PER_CALL));
          }

          const requestedIds = req.ids?.[contentType];
          const ids = requestedIds && requestedIds.length > 0
            ? requestedIds.slice(0, MAX_IDS_PER_CALL)
            : null;

          const { docs, queriedCount } = await fetchDocs(contentType, {
            session, authHeader: indexSession.authHeader, jmapAccountId, ids, windowDays,
          });
          written[contentType] = index.upsert(docs);

          if (req.prune && contentType === 'mail') {
            // Only mail prunes by date: calendar's window looks forward,
            // contacts have no date, and file rows are metadata-sized.
            // A null window means keep everything - pruning is SKIPPED, not
            // run with some fallback bound, or the setting would silently
            // delete the history the user just asked to retain.
            const cutoff = windowStartIso(windowDays);
            if (cutoff) index.pruneOlderThan(jmapAccountId, 'mail', cutoff);
          }

          // Contact/file DELETES: a JMAP `destroyed` only ever reaches this
          // route via `req.removed`, which nothing in the renderer populates
          // today - so without this, a deleted contact or file stays
          // searchable (and retrievable by the AI feature) forever. Mail and
          // calendar can't use the same trick: their queries are windowed by
          // date, so an id missing from one fetch may simply be outside the
          // window, not gone. Contacts/files have no date filter at all - the
          // query is "the first N, capped" - so when a catch-up fetch (ids
          // === null) comes back under the cap, it IS the complete set, and
          // anything indexed but absent from it is safely known to be deleted.
          if (queriedCount !== undefined && (contentType === 'contact' || contentType === 'file')) {
            const cap = contentType === 'contact' ? CONTACTS_MAX : FILES_MAX;
            const stale = strayIdsAfterCatchUp(
              index.existingIds(jmapAccountId, contentType),
              docs.map((d) => d.id),
              queriedCount,
              cap,
            );
            if (stale.length > 0) index.remove(jmapAccountId, contentType, stale);
          }
        } catch (error) {
          // One unsupported or misbehaving type must not fail the others.
          const message = error instanceof Error ? error.message : String(error);
          errors.push({ contentType, message });
          if (error instanceof JmapIndexError && error.status === 401) throw error;
        }
      }
    } finally {
      index.close();
    }
  });

  const result: IndexResult = {
    accountId: indexSession.accountId,
    written,
    skipped: notAttempted,
    errors,
    durationMs: Date.now() - started,
  };
  logger.info('mail-index: pass complete', {
    slot: indexSession.slot,
    written: JSON.stringify(written),
    skipped: notAttempted.join(',') || 'none',
    errors: errors.length,
    durationMs: result.durationMs,
  });
  return result;
}