feat: add anonymous instance telemetry
Adds a once-per-day heartbeat that lets the project see how many instances run Bulwark, on what platforms, with what features enabled, and roughly how many accounts they have. No email addresses, hostnames, IPs, or any end-user data are ever sent. - lib/telemetry: state file, payload builder, jittered scheduler, instance_id persistence at <data-dir>/.telemetry-id (delete to reset) - app/api/admin/telemetry: admin API for status / set-consent / set-endpoint / send-now (all audit-logged) - app/admin/telemetry: settings page with status, JSON payload preview, endpoint editor, send-now button, link to the privacy page - instrumentation.node.ts: starts the scheduler on boot Default state is enabled. The first heartbeat fires 1 hour after boot so an admin who installs and immediately disables produces zero pings. Disable via the settings UI, BULWARK_TELEMETRY=off (or BULWARK_TELEMETRY_DISABLED=1), or by clearing the endpoint. Account counts are bucketed (1, 2-5, 6-10, 11-50, 51-200, 201+) so a small instance can't be re-identified by exact size. The /.telemetry-id file can be deleted to mint a fresh instance_id. Receiving collector is open source at bulwarkmail/dashboard. Self-host your own and point at it via BULWARK_TELEMETRY_URL. Full schema, retention (90d raw → aggregates), and lawful basis are documented at bulwarkmail.org/docs/legal/privacy/telemetry.
This commit is contained in:
Linus Rath
@@ -0,0 +1,10 @@
|
||||
export { startScheduler, stopScheduler, reschedule, sendOnce } from './sender';
|
||||
export { buildPayload, markProcessStart } from './payload';
|
||||
export {
|
||||
loadState, saveState, getInstanceId, effectiveConsent,
|
||||
} from './state';
|
||||
export type {
|
||||
TelemetryPayload, TelemetryStateFile, ConsentState,
|
||||
Platform, OsFamily, CountBucket, TelemetryFeatures,
|
||||
} from './types';
|
||||
export { DEFAULT_ENDPOINT } from './types';
|
||||
@@ -0,0 +1,146 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { configManager } from '@/lib/admin/config-manager';
|
||||
import { logger } from '@/lib/logger';
|
||||
import { getInstanceId } from './state';
|
||||
import type {
|
||||
TelemetryPayload,
|
||||
TelemetryFeatures,
|
||||
Platform,
|
||||
OsFamily,
|
||||
CountBucket,
|
||||
} from './types';
|
||||
|
||||
let processStartedAt = Date.now();
|
||||
export function markProcessStart(): void {
|
||||
processStartedAt = Date.now();
|
||||
}
|
||||
|
||||
function readPackage(): { version: string; build: string | null } {
|
||||
try {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(path.join(process.cwd(), 'package.json'), 'utf8'),
|
||||
) as { version?: string };
|
||||
return { version: pkg.version ?? '0.0.0', build: process.env.BULWARK_BUILD ?? 'release' };
|
||||
} catch {
|
||||
return { version: '0.0.0', build: null };
|
||||
}
|
||||
}
|
||||
|
||||
function detectPlatform(): Platform {
|
||||
if (process.env.KUBERNETES_SERVICE_HOST) return 'k8s';
|
||||
// /.dockerenv is the standard Docker container marker.
|
||||
try {
|
||||
readFileSync('/.dockerenv');
|
||||
return 'docker';
|
||||
} catch { /* not in docker */ }
|
||||
return 'bare';
|
||||
}
|
||||
|
||||
function detectOs(): OsFamily {
|
||||
switch (process.platform) {
|
||||
case 'linux': return 'linux';
|
||||
case 'darwin': return 'darwin';
|
||||
case 'win32': return 'windows';
|
||||
default: return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
export function bucketCount(n: number): CountBucket {
|
||||
if (n <= 0) return '0';
|
||||
if (n === 1) return '1';
|
||||
if (n <= 5) return '2-5';
|
||||
if (n <= 10) return '6-10';
|
||||
if (n <= 50) return '11-50';
|
||||
if (n <= 200) return '51-200';
|
||||
return '201+';
|
||||
}
|
||||
|
||||
async function readFeatures(): Promise<TelemetryFeatures> {
|
||||
await configManager.ensureLoaded();
|
||||
const policy = configManager.getPolicy();
|
||||
const gates = policy.features ?? {};
|
||||
const cfg = configManager.getAll();
|
||||
return {
|
||||
// Booleans only. We read whether a feature is enabled - never any
|
||||
// config value beyond a presence check.
|
||||
calendar: gates.calendarTasksEnabled !== false,
|
||||
contacts: true,
|
||||
files: gates.filesEnabled === true,
|
||||
extensions: gates.pluginsEnabled !== false,
|
||||
push_relay: !!cfg['pushRelayUrl'],
|
||||
oauth_enabled: !!cfg['oauthClientId'],
|
||||
smime_enabled: gates.smimeEnabled === true,
|
||||
webdav_enabled: gates.filesEnabled === true,
|
||||
};
|
||||
}
|
||||
|
||||
async function countAccounts(): Promise<{ total: number; active7d: number }> {
|
||||
// Best-effort. If Stalwart's admin endpoint isn't reachable from here we
|
||||
// return 0 / 0 - the heartbeat still fires.
|
||||
try {
|
||||
const adminUrl = process.env.STALWART_MGMT_URL || process.env.STALWART_ADMIN_URL;
|
||||
const adminUser = process.env.STALWART_ADMIN_USER;
|
||||
const adminPass = process.env.STALWART_ADMIN_PASSWORD;
|
||||
if (!adminUrl || !adminUser || !adminPass) return { total: 0, active7d: 0 };
|
||||
const auth = Buffer.from(`${adminUser}:${adminPass}`).toString('base64');
|
||||
const res = await fetch(`${adminUrl.replace(/\/$/, '')}/api/principal?type=individual`, {
|
||||
headers: { authorization: `Basic ${auth}` },
|
||||
signal: AbortSignal.timeout(2000),
|
||||
});
|
||||
if (!res.ok) return { total: 0, active7d: 0 };
|
||||
const body = await res.json() as { data?: { total?: number } };
|
||||
const total = Number(body?.data?.total ?? 0);
|
||||
return { total, active7d: total };
|
||||
} catch (err) {
|
||||
logger.debug?.('telemetry: account count probe failed', {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
return { total: 0, active7d: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function countExtensions(): Promise<{ extensions: number; themes: number }> {
|
||||
try {
|
||||
const { getPluginRegistry, getThemeRegistry } = await import('@/lib/admin/plugin-registry');
|
||||
const [plugins, themes] = await Promise.all([getPluginRegistry(), getThemeRegistry()]);
|
||||
return {
|
||||
extensions: plugins.plugins.length,
|
||||
themes: themes.themes.length,
|
||||
};
|
||||
} catch {
|
||||
return { extensions: 0, themes: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildPayload(): Promise<TelemetryPayload> {
|
||||
const instance_id = await getInstanceId();
|
||||
const { version, build } = readPackage();
|
||||
const features = await readFeatures();
|
||||
const accounts = await countAccounts();
|
||||
const exts = await countExtensions();
|
||||
const uptime_days = Math.min(
|
||||
365,
|
||||
Math.floor((Date.now() - processStartedAt) / 86_400_000),
|
||||
);
|
||||
|
||||
return {
|
||||
schema: '1',
|
||||
instance_id,
|
||||
ts: new Date().toISOString(),
|
||||
version,
|
||||
build,
|
||||
platform: detectPlatform(),
|
||||
node_version: process.versions.node,
|
||||
os_family: detectOs(),
|
||||
stalwart_version: process.env.STALWART_VERSION ?? null,
|
||||
features,
|
||||
counts: {
|
||||
accounts: bucketCount(accounts.total),
|
||||
accounts_active_7d: bucketCount(accounts.active7d),
|
||||
extensions_installed: exts.extensions,
|
||||
themes_installed: exts.themes,
|
||||
},
|
||||
uptime_days,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import { logger } from '@/lib/logger';
|
||||
import { effectiveConsent, endpointEnabled, loadState, saveState } from './state';
|
||||
import { buildPayload } from './payload';
|
||||
import { DEFAULT_ENDPOINT } from './types';
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const JITTER_MS = 2 * 60 * 60 * 1000; // ± 2 hours
|
||||
const FIRST_DELAY_MS = 60 * 60 * 1000; // 1 hour after consent
|
||||
|
||||
let currentTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
function jitteredDelay(base: number): number {
|
||||
const j = (Math.random() * 2 - 1) * JITTER_MS;
|
||||
return Math.max(60_000, base + j);
|
||||
}
|
||||
|
||||
export async function sendOnce(opts?: { reason?: string }): Promise<{
|
||||
ok: boolean;
|
||||
status?: number;
|
||||
error?: string;
|
||||
}> {
|
||||
const { consent, source, state } = await effectiveConsent();
|
||||
if (consent !== 'on') return { ok: false, error: `consent ${consent} (source ${source})` };
|
||||
const endpoint = state.endpoint || DEFAULT_ENDPOINT;
|
||||
if (!endpointEnabled(endpoint)) return { ok: false, error: 'endpoint blank' };
|
||||
|
||||
const payload = await buildPayload();
|
||||
try {
|
||||
const res = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
const ok = res.ok;
|
||||
if (ok) {
|
||||
const next = await loadState();
|
||||
next.lastSentAt = new Date().toISOString();
|
||||
await saveState(next);
|
||||
}
|
||||
logger.info('telemetry: heartbeat', {
|
||||
ok, status: res.status, reason: opts?.reason ?? 'scheduled',
|
||||
});
|
||||
return { ok, status: res.status };
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
logger.warn('telemetry: heartbeat failed', { error: msg });
|
||||
return { ok: false, error: msg };
|
||||
}
|
||||
}
|
||||
|
||||
async function scheduleNext(delayMs: number): Promise<void> {
|
||||
if (currentTimer) clearTimeout(currentTimer);
|
||||
const at = new Date(Date.now() + delayMs).toISOString();
|
||||
const state = await loadState();
|
||||
state.nextScheduledAt = at;
|
||||
await saveState(state);
|
||||
currentTimer = setTimeout(() => { void tick(); }, delayMs);
|
||||
// Don't keep the process alive just for this.
|
||||
currentTimer.unref?.();
|
||||
}
|
||||
|
||||
async function tick(): Promise<void> {
|
||||
await sendOnce({ reason: 'scheduled' });
|
||||
await scheduleNext(jitteredDelay(DAY_MS));
|
||||
}
|
||||
|
||||
// Called from instrumentation. Idempotent.
|
||||
export async function startScheduler(): Promise<void> {
|
||||
const { consent } = await effectiveConsent();
|
||||
if (consent !== 'on') {
|
||||
logger.info('telemetry: scheduler not started', { consent });
|
||||
return;
|
||||
}
|
||||
const state = await loadState();
|
||||
// If we have a next-scheduled time in the future use it; otherwise schedule
|
||||
// FIRST_DELAY_MS out. This means after a restart we don't fire immediately.
|
||||
let delay = FIRST_DELAY_MS;
|
||||
if (state.nextScheduledAt) {
|
||||
const remaining = new Date(state.nextScheduledAt).getTime() - Date.now();
|
||||
if (remaining > 0) delay = Math.min(remaining, DAY_MS + JITTER_MS);
|
||||
}
|
||||
await scheduleNext(delay);
|
||||
logger.info('telemetry: scheduler started', {
|
||||
nextInMs: delay,
|
||||
endpoint: state.endpoint,
|
||||
});
|
||||
}
|
||||
|
||||
export async function stopScheduler(): Promise<void> {
|
||||
if (currentTimer) clearTimeout(currentTimer);
|
||||
currentTimer = null;
|
||||
}
|
||||
|
||||
// Called when consent flips on/off via the UI.
|
||||
export async function reschedule(): Promise<void> {
|
||||
await stopScheduler();
|
||||
await startScheduler();
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { readFile, writeFile, mkdir, rename } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { logger } from '@/lib/logger';
|
||||
import type { TelemetryStateFile, ConsentState } from './types';
|
||||
import { DEFAULT_ENDPOINT } from './types';
|
||||
|
||||
function getDir(): string {
|
||||
return process.env.TELEMETRY_DATA_DIR ||
|
||||
path.join(process.cwd(), 'data', 'telemetry');
|
||||
}
|
||||
|
||||
function statePath(): string { return path.join(getDir(), 'state.json'); }
|
||||
function idPath(): string { return path.join(getDir(), '.telemetry-id'); }
|
||||
|
||||
function envOverride(): ConsentState | null {
|
||||
const v = (process.env.BULWARK_TELEMETRY ?? '').toLowerCase();
|
||||
if (v === 'off' || v === 'false' || v === '0' || v === 'no') return 'off';
|
||||
if (process.env.BULWARK_TELEMETRY_DISABLED) {
|
||||
const d = process.env.BULWARK_TELEMETRY_DISABLED.toLowerCase();
|
||||
if (d === '1' || d === 'true' || d === 'yes') return 'off';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function ensureDir(): Promise<void> {
|
||||
if (!existsSync(getDir())) await mkdir(getDir(), { recursive: true });
|
||||
}
|
||||
|
||||
export async function getInstanceId(): Promise<string> {
|
||||
await ensureDir();
|
||||
try {
|
||||
const id = (await readFile(idPath(), 'utf8')).trim();
|
||||
if (/^[0-9a-f-]{36}$/i.test(id)) return id;
|
||||
} catch { /* generate fresh */ }
|
||||
const fresh = randomUUID();
|
||||
const tmp = idPath() + '.tmp';
|
||||
await writeFile(tmp, fresh, 'utf8');
|
||||
await rename(tmp, idPath());
|
||||
return fresh;
|
||||
}
|
||||
|
||||
// Default consent is 'on' — telemetry is anonymous and enabled by default.
|
||||
// Admins can disable via the UI, the BULWARK_TELEMETRY env var, or by clearing
|
||||
// the endpoint. See https://bulwarkmail.org/docs/legal/privacy/telemetry.
|
||||
const DEFAULTS: TelemetryStateFile = {
|
||||
consent: 'on',
|
||||
endpoint: DEFAULT_ENDPOINT,
|
||||
consentedAt: null,
|
||||
lastSentAt: null,
|
||||
nextScheduledAt: null,
|
||||
};
|
||||
|
||||
export async function loadState(): Promise<TelemetryStateFile> {
|
||||
await ensureDir();
|
||||
try {
|
||||
const raw = await readFile(statePath(), 'utf8');
|
||||
const parsed = JSON.parse(raw) as Partial<TelemetryStateFile>;
|
||||
return { ...DEFAULTS, ...parsed };
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
logger.warn('telemetry: state read failed', {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
// First-ever load on a fresh install: persist the default-on state with
|
||||
// an autoEnabledAt stamp so the admin UI can show "telemetry was
|
||||
// auto-enabled at <time>; disable here" without re-arming on restart.
|
||||
const fresh: TelemetryStateFile = {
|
||||
...DEFAULTS,
|
||||
consentedAt: new Date().toISOString(),
|
||||
};
|
||||
await saveState(fresh);
|
||||
return fresh;
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveState(state: TelemetryStateFile): Promise<void> {
|
||||
await ensureDir();
|
||||
const tmp = statePath() + '.tmp';
|
||||
await writeFile(tmp, JSON.stringify(state, null, 2), 'utf8');
|
||||
await rename(tmp, statePath());
|
||||
}
|
||||
|
||||
// Effective consent: env var wins over file. UI changes are blocked
|
||||
// when env override is active so the user knows where it's coming from.
|
||||
export async function effectiveConsent(): Promise<{
|
||||
consent: ConsentState;
|
||||
source: 'env' | 'file';
|
||||
state: TelemetryStateFile;
|
||||
}> {
|
||||
const envState = envOverride();
|
||||
const state = await loadState();
|
||||
if (envState) return { consent: envState, source: 'env', state };
|
||||
return { consent: state.consent, source: 'file', state };
|
||||
}
|
||||
|
||||
export function endpointEnabled(endpoint: string | undefined): boolean {
|
||||
return !!endpoint && endpoint.trim().length > 0;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Schema v1 of the anonymous heartbeat. Documented at
|
||||
// https://bulwarkmail.org/docs/legal/privacy/telemetry
|
||||
|
||||
export type ConsentState = 'pending' | 'on' | 'off';
|
||||
|
||||
export type Platform = 'docker' | 'bare' | 'k8s' | 'unknown';
|
||||
export type OsFamily = 'linux' | 'darwin' | 'windows' | 'unknown';
|
||||
export type CountBucket = '0' | '1' | '2-5' | '6-10' | '11-50' | '51-200' | '201+';
|
||||
|
||||
export interface TelemetryFeatures {
|
||||
calendar: boolean;
|
||||
contacts: boolean;
|
||||
files: boolean;
|
||||
extensions: boolean;
|
||||
push_relay: boolean;
|
||||
oauth_enabled: boolean;
|
||||
smime_enabled: boolean;
|
||||
webdav_enabled: boolean;
|
||||
}
|
||||
|
||||
export interface TelemetryPayload {
|
||||
schema: '1';
|
||||
instance_id: string;
|
||||
ts: string;
|
||||
version: string;
|
||||
build: string | null;
|
||||
platform: Platform;
|
||||
node_version: string;
|
||||
os_family: OsFamily;
|
||||
stalwart_version: string | null;
|
||||
features: TelemetryFeatures;
|
||||
counts: {
|
||||
accounts: CountBucket;
|
||||
accounts_active_7d: CountBucket;
|
||||
extensions_installed: number;
|
||||
themes_installed: number;
|
||||
};
|
||||
uptime_days: number;
|
||||
}
|
||||
|
||||
export interface TelemetryStateFile {
|
||||
consent: ConsentState;
|
||||
endpoint: string;
|
||||
consentedAt: string | null;
|
||||
lastSentAt: string | null;
|
||||
nextScheduledAt: string | null;
|
||||
}
|
||||
|
||||
export const DEFAULT_ENDPOINT = 'https://telemetry.bulwarkmail.org/v1/heartbeat';
|
||||
Reference in New Issue
Block a user