SRCmail/integration/tests/13-electron-offline-replica.spec.ts

import { test, expect, _electron as electron } from '@playwright/test';
import type { ElectronApplication, Page } from '@playwright/test';
import { spawn, type ChildProcess } from 'node:child_process';
import { createServer } from 'node:net';
import { get as httpGet } from 'node:http';
import { createHash, randomBytes } from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { ACCOUNTS, JMAP_URL } from './helpers/config';
import { sendMail } from './helpers/smtp';
import { JmapClient } from './helpers/jmap';

/**
 * The offline mail replica (lib/offline-replica/**) against the real Stalwart
 * fixture, with a REAL NETWORK CUT.
 *
 * THE POINT OF THIS FILE: a sync test that never tests the offline case has not
 * tested the feature. So test 1 syncs against a live server, then makes the
 * backend genuinely unreachable, and only then asserts that a previously-synced
 * message still returns its full HTML body - from the encrypted replica, with no
 * network available to fall back to.
 *
 * HOW THE CUT IS MADE. The standalone server is started with
 * `JMAP_SERVER_URL` pointing at a LOCAL PROXY that forwards to Stalwart. Killing
 * the proxy's listener makes every JMAP request fail with ECONNREFUSED - a real
 * transport failure at the socket level, not a mock, not a stubbed fetch, and not
 * a flag the code under test can see. Preferred over stopping the Stalwart
 * container because it cuts only THIS test's path and leaves the shared fixture
 * (and any concurrently-running suite) untouched.
 *
 * THE SAME TWO CONSTRAINTS as 12-electron-mail-index.spec.ts apply and are why
 * this is split into two tests rather than one:
 *
 *  1. The RENDERER cannot reach this fixture from a production build. It talks
 *     JMAP directly to Stalwart, which here is deliberately plain HTTP, and the
 *     production CSP pins `connect-src` to `'self' https: wss:`. NODE_ENV at
 *     runtime does not help - `next build` inlines it into the middleware.
 *  2. The fd-3 key channel cannot survive `next dev`, which claims fd 3 for its
 *     own IPC. So the two configurations are mutually exclusive: a real key
 *     channel means no browser, a browser means no key channel.
 *
 * Test 1 therefore drives the REAL standalone server over HTTP from Node with a
 * real fd-3 key channel - no browser needed, because the routes are the thing
 * being proven. Test 2 launches the REAL Electron shell to prove the routes exist
 * and are reachable in a genuine build, which is the class of failure only a real
 * build reveals (the standalone output silently dropping a native prebuild, say).
 *
 * WHAT THIS FILE DOES NOT PROVE: that `components/email/email-viewer.tsx` paints
 * the replica-served body in a browser while offline. That needs a renderer, a
 * key channel and a reachable-then-unreachable JMAP server simultaneously, which
 * constraints 1 and 2 make impossible against this fixture. The read path returns
 * a field-for-field `Email` (asserted below, including `bodyValues` keyed by the
 * same partIds as `htmlBody`), and the renderer-side gate is covered by
 * `lib/__tests__/offline-fallback-client.test.ts` - but the final paint is NOT
 * covered by a real offline browser run. Stated rather than implied.
 */
const alice = ACCOUNTS.alice;
const projectRoot = path.resolve(__dirname, '../..');

/** Mirrors lib/mail-index/paths.ts's accountFileToken(). */
function accountFileToken(accountId: string): string {
  return createHash('sha256').update(accountId, 'utf8').digest('hex').slice(0, 32);
}

function getFreePort(): Promise<number> {
  return new Promise((resolve, reject) => {
    const server = createServer();
    server.unref();
    server.on('error', reject);
    server.listen(0, '127.0.0.1', () => {
      const address = server.address();
      if (address && typeof address === 'object') {
        const { port } = address;
        server.close(() => resolve(port));
      } else {
        server.close(() => reject(new Error('Could not allocate a free localhost port')));
      }
    });
  });
}

function waitForServerReady(url: string, timeoutMs: number): Promise<void> {
  const deadline = Date.now() + timeoutMs;
  return new Promise((resolve, reject) => {
    const attempt = () => {
      const req = httpGet(url, (res) => {
        res.resume();
        resolve();
      });
      req.on('error', () => {
        if (Date.now() > deadline) {
          reject(new Error(`Server never became reachable at ${url}`));
          return;
        }
        setTimeout(attempt, 300);
      });
    };
    attempt();
  });
}

/**
 * A raw TCP forwarder in front of Stalwart, so the test can sever the backend at
 * the socket level. `cut()` closes the listener AND destroys every live socket, so
 * a pooled keep-alive connection cannot keep working after the cut.
 */
async function startCuttableProxy(target: { host: string; port: number }): Promise<{
  port: number;
  cut: () => Promise<void>;
  stop: () => Promise<void>;
}> {
  const { connect } = await import('node:net');
  const sockets = new Set<import('node:net').Socket>();
  const server = createServer((incoming) => {
    sockets.add(incoming);
    incoming.on('close', () => sockets.delete(incoming));
    incoming.on('error', () => incoming.destroy());
    const upstream = connect(target.port, target.host, () => {
      incoming.pipe(upstream);
      upstream.pipe(incoming);
    });
    sockets.add(upstream);
    upstream.on('close', () => sockets.delete(upstream));
    upstream.on('error', () => { incoming.destroy(); upstream.destroy(); });
  });
  const port = await getFreePort();
  await new Promise<void>((resolve, reject) => {
    server.once('error', reject);
    server.listen(port, '127.0.0.1', () => resolve());
  });

  const closeAll = () =>
    new Promise<void>((resolve) => {
      for (const s of sockets) s.destroy();
      sockets.clear();
      server.close(() => resolve());
      // `close()` only stops new connections; the destroys above handle the rest.
      setTimeout(resolve, 500);
    });

  return { port, cut: closeAll, stop: closeAll };
}

/**
 * Serves the key protocol of electron/key-service.ts over the child's inherited
 * fd. The key and the encryption are real; only safeStorage's wrapping of it is
 * out of the picture here, which is what test 2 covers.
 */
function serveKeyChannel(child: ChildProcess, fdIndex: number, key: Buffer): void {
  const channel = child.stdio[fdIndex] as NodeJS.ReadWriteStream | null;
  if (!channel) throw new Error(`child has no fd ${fdIndex} - key channel missing`);
  let buffer = '';
  channel.on('data', (chunk: Buffer) => {
    buffer += chunk.toString('utf8');
    let newline: number;
    while ((newline = buffer.indexOf('\n')) >= 0) {
      const line = buffer.slice(0, newline);
      buffer = buffer.slice(newline + 1);
      if (!line.trim()) continue;
      const req = JSON.parse(line) as { id?: number; op?: string };
      const reply =
        req.op === 'getIndexKey'
          ? { id: req.id, ok: true, key: key.toString('hex') }
          : req.op === 'deleteIndexKey'
            ? { id: req.id, ok: true }
            : { id: req.id, ok: false, code: 'bad-request', error: 'unknown op' };
      channel.write(`${JSON.stringify(reply)}\n`);
    }
  });
}

class Jar {
  private cookies = new Map<string, string>();
  absorb(response: Response): void {
    for (const raw of response.headers.getSetCookie()) {
      const [pair] = raw.split(';');
      const eq = pair.indexOf('=');
      if (eq <= 0) continue;
      this.cookies.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
    }
  }
  header(): string {
    return [...this.cookies].map(([k, v]) => `${k}=${v}`).join('; ');
  }
}

interface CycleReport {
  ok: boolean;
  unfinishedWork: boolean;
  bootstrapped: boolean;
  envelopesWritten: number;
  bodiesWritten: number;
  envelopesDeleted: number;
  coveragePhase: string;
  resyncRequired: boolean;
  warnings: string[];
  error?: string;
  errorClass?: string;
}

test.describe('Electron desktop shell - offline mail replica', () => {
  test('syncs full bodies, then serves a synced message with the backend UNREACHABLE', async () => {
    test.setTimeout(240_000);

    const jmap = await JmapClient.connect(alice.email, alice.password);
    await jmap.reset();

    const stamp = Date.now();
    const subject = `IT replica subject ${stamp}`;
    // Appears ONLY in the HTML body, so a hit proves the full body was stored -
    // not the preview or the subject, which any envelope already carries.
    const bodyPhrase = `luzernrenewal${stamp}`;
    const htmlMarker = `<strong>${bodyPhrase}</strong>`;

    await sendMail({
      from: alice.email,
      authPass: alice.password,
      to: alice.email,
      subject,
      body: `plain text ${bodyPhrase}`,
      html: `<html><body><p>Please review the ${htmlMarker} before September.</p></body></html>`,
    });
    // A second message, so "the list came from the replica" is not a one-row
    // coincidence.
    await sendMail({
      from: alice.email,
      authPass: alice.password,
      to: alice.email,
      subject: `IT replica second ${stamp}`,
      body: 'the second message',
    });

    const storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-it-'));
    const key = randomBytes(32);
    const stalwart = new URL(JMAP_URL);
    const proxy = await startCuttableProxy({
      host: stalwart.hostname,
      port: Number(stalwart.port || 80),
    });
    const proxiedJmapUrl = `http://127.0.0.1:${proxy.port}`;

    const port = await getFreePort();
    const baseUrl = `http://127.0.0.1:${port}`;
    const serverEntry = path.join(projectRoot, '.next', 'standalone', 'server.js');
    expect(
      fs.existsSync(serverEntry),
      `missing ${serverEntry} - run "npm run build:standalone" first`,
    ).toBe(true);

    const server = spawn(process.execPath, [serverEntry], {
      cwd: path.dirname(serverEntry),
      env: {
        ...process.env,
        PORT: String(port),
        HOSTNAME: '127.0.0.1',
        NODE_ENV: 'production',
        // Through the cuttable proxy, so the backend can be severed later.
        JMAP_SERVER_URL: proxiedJmapUrl,
        SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
        VNCMAIL_DESKTOP_STORE_DIR: storeDir,
        VNCMAIL_DESKTOP_KEY_FD: '3',
      },
      stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
    });
    server.stderr?.on('data', (chunk) => process.stderr.write(`[standalone] ${chunk}`));
    serveKeyChannel(server, 3, key);

    const jar = new Jar();
    const call = async (url: string, init?: RequestInit): Promise<Response> => {
      const response = await fetch(`${baseUrl}${url}`, {
        ...init,
        headers: { ...(init?.headers ?? {}), cookie: jar.header() },
      });
      jar.absorb(response);
      return response;
    };
    const sync = async (body: Record<string, unknown> = {}): Promise<CycleReport> => {
      const response = await call('/api/offline/sync', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(body),
      });
      const parsed = await response.json();
      expect(response.status, JSON.stringify(parsed)).toBe(200);
      return parsed.report as CycleReport;
    };

    try {
      await waitForServerReady(baseUrl, 90_000);

      const login = await call('/api/auth/session?slot=0', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          serverUrl: proxiedJmapUrl,
          username: alice.email,
          password: alice.password,
          slot: 0,
        }),
      });
      const loginText = await login.text();
      expect(login.status, `login failed: ${loginText}`).toBe(200);

      // The gate must be open and the native binding loaded, or every assertion
      // below would fail for an unrelated reason.
      const reachable = await call('/api/offline/status');
      const reachableText = await reachable.text();
      expect(
        reachable.status,
        `replica routes unreachable: ${reachableText.slice(0, 400)}`,
      ).toBe(200);

      // ── ONLINE: bootstrap, then chain until the cycle reports itself done ──
      const first = await sync();
      expect(first.error, `first cycle failed: ${first.error}`).toBeUndefined();
      expect(first.bootstrapped, 'the first cycle must bootstrap').toBe(true);

      let report = first;
      for (let i = 0; i < 12 && report.unfinishedWork; i++) report = await sync();
      expect(
        report.unfinishedWork,
        `sync never settled: ${JSON.stringify(report)}`,
      ).toBe(false);
      // Termination is a real property here: the body-queue give-up marks and the
      // inserted-not-attempted count are what stop this looping forever.
      expect(report.coveragePhase).toBe('complete');
      expect(report.resyncRequired).toBe(false);

      const status = await (await call('/api/offline/status')).json();
      expect(status.synced).toBe(true);
      expect(
        status.stats.envelopes,
        `no envelopes stored: ${JSON.stringify(status.stats)}`,
      ).toBeGreaterThanOrEqual(2);
      expect(
        status.stats.bodies,
        `no BODIES stored - the replica would be no better than the search index`,
      ).toBeGreaterThanOrEqual(2);
      expect(status.stats.mailboxes).toBeGreaterThan(0);

      // ── THE DELTA PATH: a message that arrives AFTER the cursor was captured ──
      // Bootstrap alone would satisfy every assertion below, so this is what actually
      // exercises `Email/changes` and proves the stored cursor is USABLE rather than
      // merely present. It is also the assertion that fails if an `Email/get` state
      // token is ever adopted as a `/changes` cursor: the fast-forwarded cursor
      // reports no changes, and this message never arrives.
      const deltaSubject = `IT replica delta ${stamp}`;
      const deltaPhrase = `bernrenewal${stamp}`;
      await sendMail({
        from: alice.email,
        authPass: alice.password,
        to: alice.email,
        subject: deltaSubject,
        body: `plain ${deltaPhrase}`,
        html: `<html><body><p>delta ${deltaPhrase}</p></body></html>`,
      });

      let delta = await sync();
      for (let i = 0; i < 10 && (delta.unfinishedWork || delta.envelopesWritten === 0); i++) {
        delta = await sync();
      }
      expect(delta.bootstrapped, 'the delta cycle must NOT re-bootstrap').toBe(false);
      const afterDelta = await (await call('/api/offline/status')).json();
      expect(
        afterDelta.stats.envelopes,
        `Email/changes did not deliver a message that arrived after the cursor was ` +
          `captured: ${JSON.stringify(afterDelta.stats)}`,
      ).toBeGreaterThanOrEqual(3);
      expect(
        afterDelta.stats.bodies,
        'the delta path delivered the envelope but never queued its body',
      ).toBeGreaterThanOrEqual(3);
      expect(afterDelta.resyncRequired, 'a healthy delta cycle must not invalidate a cursor').toBe(false);

      // Find the message and its mailbox while still online, so the offline phase
      // asserts on known ids rather than discovering them from the thing under test.
      const mailboxesOnline = await (await call('/api/offline/mail?kind=mailboxes')).json();
      const inbox = (mailboxesOnline.mailboxes as Array<{ id: string; role?: string }>)
        .find((m) => m.role === 'inbox');
      expect(inbox, 'the replica holds no inbox').toBeTruthy();

      const listOnline = await (
        await call(`/api/offline/mail?kind=list&mailboxId=${encodeURIComponent(inbox!.id)}&limit=50`)
      ).json();
      const target = (listOnline.emails as Array<{ id: string; subject?: string }>)
        .find((e) => e.subject === subject);
      expect(target, `the synced message is not in the replica: ${JSON.stringify(listOnline.emails?.map((e: {subject?: string}) => e.subject))}`).toBeTruthy();

      // ── THE CUT: sever the backend at the socket level ────────────────────
      await proxy.cut();

      // Prove the cut is real, from inside the server process's own network
      // namespace: a live JMAP call must now fail. `/api/offline/sync` reaches
      // Stalwart first thing, so it is the honest probe.
      const afterCut = await call('/api/offline/sync', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({}),
      });
      const afterCutBody = await afterCut.json();
      expect(
        afterCut.status,
        `the backend is still reachable, so the offline assertions below would prove nothing: ` +
          `${JSON.stringify(afterCutBody)}`,
      ).not.toBe(200);

      // ── OFFLINE: the actual feature ───────────────────────────────────────
      const messageResponse = await call(
        `/api/offline/mail?kind=message&id=${encodeURIComponent(target!.id)}`,
      );
      // Read the body ONCE: `expect`'s message argument is evaluated eagerly, so
      // putting `await response.text()` in it consumes the stream before .json().
      const messageText = await messageResponse.text();
      expect(
        messageResponse.status,
        `the offline read path failed with the backend down: ${messageText.slice(0, 400)}`,
      ).toBe(200);
      const offline = JSON.parse(messageText);
      expect(offline.available).toBe(true);
      expect(offline.hasBody, 'the message has no stored body offline').toBe(true);

      const email = offline.email as {
        id: string; subject?: string; receivedAt: string;
        htmlBody?: Array<{ partId: string; type: string }>;
        textBody?: Array<{ partId: string }>;
        bodyValues?: Record<string, { value: string }>;
        from?: Array<{ email: string }>;
        keywords?: Record<string, boolean>;
        mailboxIds?: Record<string, boolean>;
        headers?: Record<string, string | string[]>;
      };

      expect(email.id).toBe(target!.id);
      expect(email.subject).toBe(subject);

      // THE ASSERTION: the full HTML body, recovered with no network.
      const htmlPartId = email.htmlBody?.[0]?.partId;
      expect(htmlPartId, 'no htmlBody part offline').toBeTruthy();
      const html = email.bodyValues?.[htmlPartId as string]?.value ?? '';
      expect(
        html,
        'the HTML body is not in the replica - this is the whole feature',
      ).toContain(htmlMarker);
      expect(html).toContain(bodyPhrase);

      // `bodyValues` MUST be keyed by the same partIds as htmlBody/textBody, or
      // email-viewer.tsx's isBodyLoading gate sits on its skeleton forever
      // (hasBodyParts true, bodyValues unusable).
      for (const part of [...(email.htmlBody ?? []), ...(email.textBody ?? [])]) {
        expect(
          email.bodyValues?.[part.partId],
          `bodyValues is missing partId ${part.partId}, which the viewer requires`,
        ).toBeTruthy();
      }

      // The rest of the shape the renderer reads.
      expect(email.from?.[0]?.email).toBe(alice.email);
      expect(email.receivedAt).toBeTruthy();
      expect(Object.keys(email.mailboxIds ?? {})).toContain(inbox!.id);
      // Header normalisation happened server-side (the array -> record flattening
      // the online path does in parseEmailHeaders).
      expect(email.headers && !Array.isArray(email.headers)).toBe(true);

      // The list and the folder tree must also survive the cut.
      const listOffline = await (
        await call(`/api/offline/mail?kind=list&mailboxId=${encodeURIComponent(inbox!.id)}&limit=50`)
      ).json();
      expect(listOffline.available).toBe(true);
      expect(listOffline.emails.length).toBeGreaterThanOrEqual(2);
      expect(
        (listOffline.emails as Array<{ subject?: string }>).map((e) => e.subject),
      ).toContain(subject);

      const mailboxesOffline = await (await call('/api/offline/mail?kind=mailboxes')).json();
      expect(mailboxesOffline.available).toBe(true);
      expect((mailboxesOffline.mailboxes as unknown[]).length).toBeGreaterThan(0);

      // Status must be readable offline too - a user with no network still needs
      // to see what they have and be able to free the space.
      const statusOffline = await (await call('/api/offline/status')).json();
      expect(statusOffline.ok).toBe(true);
      expect(statusOffline.stats.bodies).toBeGreaterThanOrEqual(2);

      // A cycle attempted while offline must classify as Transport and must NOT
      // touch the data. "Offline is not an error."
      expect(
        ['Transport', 'ServerTransient'].includes(String(afterCutBody.code)),
        `an offline cycle must classify as Transport/ServerTransient so the caller retries ` +
          `rather than treating the feature as broken; got code=${afterCutBody.code} ` +
          `status=${afterCut.status} body=${JSON.stringify(afterCutBody)}`,
      ).toBe(true);
      const afterOfflineCycle = await (await call('/api/offline/status')).json();
      expect(
        afterOfflineCycle.stats.envelopes,
        'an offline cycle deleted data - a transport failure must never do that',
      ).toBe(statusOffline.stats.envelopes);
      expect(afterOfflineCycle.resyncRequired).toBe(false);

      // ── PURGE: the retention control has to actually free the space ────────
      const purge = await call('/api/offline/status', { method: 'DELETE' });
      expect(purge.status).toBe(200);
      const purged = await (await call('/api/offline/status')).json();
      expect(purged.synced).toBe(false);
      expect(purged.coveragePhase).toBe('never-run');
    } finally {
      server.kill();
      await proxy.stop();
      // Let the process release its WAL files before reading them.
      await new Promise((r) => setTimeout(r, 700));
    }

    // ── the file on disk is genuinely encrypted ─────────────────────────────
    const accountId = `${alice.email}@127.0.0.1`;
    const dbPath = path.join(storeDir, 'index', `${accountFileToken(accountId)}.db`);
    expect(fs.existsSync(dbPath), `no replica database at ${dbPath}`).toBe(true);

    const onDisk = Buffer.concat(
      ['', '-wal', '-shm']
        .map((suffix) => `${dbPath}${suffix}`)
        .filter((f) => fs.existsSync(f))
        .map((f) => fs.readFileSync(f)),
    );
    expect(onDisk.length).toBeGreaterThan(0);
    // `PRAGMA key` is a silent no-op on a non-SQLCipher binding - no error, a
    // working database, and the mail in cleartext - so every functional assertion
    // above would pass either way. These are the ones that catch it.
    expect(
      fs.readFileSync(dbPath).subarray(0, 15).toString('latin1'),
      'the replica file has a plain SQLite header - it is NOT encrypted',
    ).not.toBe('SQLite format 3');
    expect(
      onDisk.includes(bodyPhrase),
      'the message body is recoverable from the raw database bytes - not encrypted',
    ).toBe(false);
    expect(
      onDisk.includes(subject),
      'the subject is recoverable from the raw database bytes - not encrypted',
    ).toBe(false);

    fs.rmSync(storeDir, { recursive: true, force: true });
  });

  test('wiring: the real standalone boot reaches the replica routes with a real safeStorage key', async () => {
    test.setTimeout(180_000);
    // A FRESH profile is load-bearing, not hygiene: the 401 asserted below is
    // "nobody is signed in", and a leftover jmap_stalwart_ctx cookie from any
    // previous run turns it into a 200.
    const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'vncmail-replica-wiring-'));
    const electronApp: ElectronApplication = await electron.launch({
      args: [projectRoot, `--user-data-dir=${userDataDir}`],
      env: {
        ...process.env,
        JMAP_SERVER_URL: JMAP_URL,
        SESSION_SECRET: 'integration-not-a-real-secret-32-chars!!',
      },
    });

    try {
      const appWindow: Page = await electronApp.firstWindow();
      await appWindow.waitForLoadState('domcontentloaded');
      await appWindow.locator('#username').waitFor({ state: 'visible', timeout: 90_000 });

      const encryptionAvailable = await electronApp.evaluate(({ safeStorage }) =>
        safeStorage.isEncryptionAvailable(),
      );
      expect(
        encryptionAvailable,
        'safeStorage reports no encryption available, so main.ts correctly disabled the ' +
          'feature - this assertion cannot pass here',
      ).toBe(true);

      // 401 = the gate opened, the native binding loaded from the REAL standalone
      // artifact, and the fd-3 key channel is present; it refuses only because
      // nobody is signed in.
      //   404 => VNCMAIL_DESKTOP_STORE_DIR was never set
      //   503 => the native binding or the key channel is missing from the real
      //          build - the class of failure only a real build reveals
      for (const route of [
        '/api/offline/status',
        '/api/offline/mail?kind=mailboxes',
        '/api/offline/sync',
      ]) {
        const probe = await appWindow.evaluate(async (url) => {
          const response = await fetch(url, {
            method: url.endsWith('/sync') ? 'POST' : 'GET',
          });
          return { status: response.status, body: (await response.text()).slice(0, 300) };
        }, route);
        expect(
          probe.status,
          `${route}: expected 401 (reachable, unauthenticated) but got ${probe.status}: ${probe.body}`,
        ).toBe(401);
      }
    } finally {
      await electronApp.close();
      fs.rmSync(userDataDir, { recursive: true, force: true });
    }
  });
});