diff --git a/.agents/skills/performance-engineering/SKILL.md b/.agents/skills/performance-engineering/SKILL.md index eeef8359..1b1eb6f7 100644 --- a/.agents/skills/performance-engineering/SKILL.md +++ b/.agents/skills/performance-engineering/SKILL.md @@ -197,13 +197,19 @@ A cache inside an `O(consumers × entities × candidates)` loop is a mitigation, ## Repository Tooling -Three unattended capture commands exist; prefer them over ad-hoc timing code, +`scripts/perf/DOCUMENTATION.md` is the entry point: it covers every capture +command, how to stand up a production build to measure against, how to read the +artifacts, and the validity guarantees these scripts enforce. Read it before +measuring. + +Four unattended capture commands exist; prefer them over ad-hoc timing code, and extend them when a scenario is missing rather than measuring by hand. | Command | Answers | |---|---| | `bun run profile:idle` | What the app does while nobody interacts with it. Supports `--session`, `--tab`, `--then-tab`, `--panel`, `--expand-projects` to reach a specific mounted state, plus `--baseline` and `--budget-*` for regression gating. | | `bun run profile:session` | What a streaming assistant response costs. Creates a session, dispatches a prompt through the `openchamber session` CLI, and records until the session reports idle. Reports the long-task distribution, a timeline-trace breakdown, running animations, and output-normalised metrics. | +| `bun run profile:animation` | What a CSS animation costs, isolated from the app. Animate only `transform` and `opacity`; everything else recalculates style every frame. | | `bun run profile:browser` | A manually driven capture when the interaction cannot be scripted. | Both automated commands fail loudly rather than reporting a clean result when diff --git a/.agents/skills/theme-system/SKILL.md b/.agents/skills/theme-system/SKILL.md index 028d3a92..0430e1d3 100644 --- a/.agents/skills/theme-system/SKILL.md +++ b/.agents/skills/theme-system/SKILL.md @@ -71,8 +71,34 @@ import { Icon } from '@/components/icon/Icon'; Use `IconName` for icon values stored in arrays, objects, state, or config. `Icon` has no `size` prop. Run `bun run icons:generate` when introducing a sprite name, and never edit `sprite.ts` manually. Load `references/icons.md` for the complete workflow. +## Animation Contract + +Animate only `transform` and `opacity`. The compositor drives those; every other +property recalculates style on each frame for as long as the animation runs, and +geometry properties add layout on top. Measured on this repository's fixture, +identical at any element count from 1 to 32: + +| Animated property | Style recalculations/sec | Layouts/sec | +|---|---|---| +| `transform`, `opacity`, `filter` | 0 | 0 | +| `rotate` (the individual property) | 60 | 0 | +| `background-position`, `border-color`, `box-shadow` | 60 | 0 | +| `width` and other geometry | 60 | 60 | + +- `rotate: 360deg` is not a cheap synonym for `transform: rotate(360deg)`. + Prefer the `transform` form. +- Cost applies for the entire time an animation runs, so an indicator tied to a + long-running operation pays it continuously. An indicator that is not + conveying anything should not be animating. +- `will-change`, wrapper elements, `contain`, and `steps()` timing do not make a + non-composited property cheap. Only changing the property does. +- Verify with `bun run profile:animation` rather than reasoning about it; add a + variant to `scripts/perf/animation-fixture.html` for a technique not covered. + See `scripts/perf/DOCUMENTATION.md`. + ## Verification +- Animations are limited to `transform` and `opacity`, or their cost was measured and accepted. - No hardcoded/palette colors were introduced. - Buttons use shared variants and sizes. - Icons use `Icon`/`IconName`, and generated sprite changes are intentional. diff --git a/AGENTS.md b/AGENTS.md index 270e0a00..968cccc4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,6 +65,7 @@ High-value anchors: - Sync: `packages/ui/src/sync/DOCUMENTATION.md` - Stores: `packages/ui/src/stores/DOCUMENTATION.md` - CLI: `packages/web/bin/lib/DOCUMENTATION.md` +- Performance measurement tooling: `scripts/perf/DOCUMENTATION.md` - VS Code runtime: `packages/vscode/src/DOCUMENTATION.md` - Electron: `packages/electron/README.md` - Mobile: `packages/mobile/README.md` diff --git a/package.json b/package.json index b405a05d..1971ea11 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,8 @@ "release:test:intel": "./scripts/test-release-build.sh x86_64", "release:test:arm": "./scripts/test-release-build.sh aarch64", "profile:idle": "node scripts/profile-idle.mjs", - "profile:session": "node scripts/profile-session.mjs" + "profile:session": "node scripts/profile-session.mjs", + "profile:animation": "node scripts/profile-animation.mjs" }, "dependencies": { "@base-ui/react": "^1.4.0", diff --git a/scripts/perf/DOCUMENTATION.md b/scripts/perf/DOCUMENTATION.md new file mode 100644 index 00000000..1303eb4c --- /dev/null +++ b/scripts/perf/DOCUMENTATION.md @@ -0,0 +1,170 @@ +# Performance Measurement Tooling + +Owns the unattended performance capture commands and their shared Chrome +DevTools Protocol plumbing. Read this before measuring OpenChamber performance +or extending these scripts. The methodology rules they enforce come from +`.agents/skills/performance-engineering/SKILL.md`. + +## Commands + +| Command | Answers | +|---|---| +| `bun run profile:idle` | What the app does while nobody interacts with it. | +| `bun run profile:session` | What receiving and rendering a live assistant response costs. | +| `bun run profile:animation` | What a CSS animation costs, isolated from the app. | +| `bun run profile:browser` | A manually driven capture, for interactions that cannot be scripted. | + +All of them measure a real browser over CDP. Pass `--help` to any of them for +the full option list. + +## Before Measuring Anything + +**Measure a production build.** A development build's render and bundle +behaviour does not represent what users run. + +```bash +bun run build:ui && bun run build:web +cd && node /packages/web/bin/cli.js serve --port 4599 --foreground +``` + +`profile:idle` and `profile:session` need a running server; `profile:animation` +serves its own fixture and needs nothing. + +## profile:idle + +Loads the app, lets it settle, then records a window during which no input is +delivered. Everything it reports is therefore work the app performs while the +user is doing nothing — the class of regression users notice as fan noise, +battery drain, and a permanently busy tab. + +Reports per second of idle time: main-thread busy time, script, style +recalculation and layout time and counts, DOM node / document / frame / +listener growth, heap trajectory including a least-squares growth rate, a CPU +sampling profile with self time per function, and attribution of timer, +animation-frame and observer work to the call site that scheduled it. + +```bash +# Baseline, then compare a change against it and fail on a budget. +bun run profile:idle -- --url http://127.0.0.1:4599 --output artifacts/before +bun run profile:idle -- --url http://127.0.0.1:4599 --baseline artifacts/before --budget-cpu 5 +``` + +Scenario options reach a specific mounted state, because idle cost depends on +what is mounted: `--session`, `--tab`, `--panel `, `--expand-projects`, +`--expand-sessions`, and `--then-tab` (navigate away after settling, to measure +what a surface keeps doing once the user has left it). + +## profile:session + +Creates a session, opens it in a browser, dispatches a prompt through the +supported `openchamber session` CLI, and records until the session reports +itself idle. No input is synthesised; the prompt is the only stimulus. + +Streaming is judged by responsiveness, not totals, so the report leads with the +long-task distribution, a timeline-trace breakdown naming where time went, +running animations, the application's own stream counters, and output-normalised +metrics. + +```bash +bun run profile:session -- --url http://127.0.0.1:4599 --dir +# What an idle session costs while a different session is active elsewhere: +bun run profile:session -- --view-session --expand-projects --expand-sessions +``` + +This command calls a real model. Use a cheap one; `--model` overrides the +configured selection. + +## profile:animation + +Serves an isolated fixture and measures each animation variant directly, so a +comparison takes seconds instead of an application rebuild plus a streamed +response. + +```bash +bun run profile:animation +bun run profile:animation -- --variant border-color --count 8 +``` + +Measured on this repository's fixture, at any element count from 1 to 32: + +| Animated property | Style recalculations/sec | Layouts/sec | +|---|---|---| +| none | 0 | 0 | +| `transform` (rotate, translate, scale) | 0 | 0 | +| `opacity`, `filter` | 0 | 0 | +| `rotate` (the individual property) | 60 | 0 | +| `background-position` | 60 | 0 | +| `border-color` | 60 | 0 | +| `box-shadow` | 60 | 0 | +| `width` | 60 | 60 | + +Animate `transform` and `opacity`. Anything else recalculates style on every +frame for as long as the animation runs, and geometry properties add layout on +top. Note that `rotate: 360deg` is *not* equivalent to +`transform: rotate(360deg)` in cost. + +Add a variant to `animation-fixture.html` to measure a property or technique +that is not listed. + +## Reading The Results + +Every run writes a JSON summary next to any raw capture, so results can be +compared later without re-running: + +- `profile:idle` → `idle-summary.json`, `cpu-profile.cpuprofile` +- `profile:session` → `session-summary.json`, `cpu-profile.cpuprofile` + +`--baseline ` prints a per-metric delta table against a previous run +of the same command. `--budget-*` options make the command exit non-zero, so the +same invocation works as an investigation tool and as a regression gate. + +Artifacts can reveal project paths and endpoint names. They are gitignored; do +not publish them without review. + +## Validity Guarantees + +These commands fail loudly rather than reporting a clean result, because each +of these failure modes once produced a confident, wrong "everything is fast": + +- **Throttled renderer.** Chrome stops producing frames and throttles timers for + windows it considers backgrounded or occluded. Launch flags disable that, and + every run measures frame liveness and warns when the renderer was not + producing frames. +- **Missing trace data.** `RunTask` is only emitted under the + disabled-by-default timeline category. A capture without it would report zero + long tasks; the missing-task case is reported instead. +- **A scenario that never ran.** A session belonging to a directory the browser + is not viewing renders nothing and produces a perfectly quiet profile. + `profile:session` verifies both new message elements in the DOM and + message-list render counters before believing a quiet result. + +Preserve this property when extending these scripts. A metric reading zero must +be a measurement, never a disabled instrument. + +## Methodology Rules + +- **Never report an "after" without a "before" on the identical scenario and + build.** Rebuild the unchanged version and re-run it, however inconvenient. + Expect plausible fixes to change nothing. +- **A sampling profiler cannot explain native work.** Self time in `(program)` + only means the time was not in interpreted JavaScript. Use the trace + breakdown, which names parsing, style, layout, layerization, paint and raster. +- **Normalise when the workload varies.** Assistant responses differ in length + between runs, so per-second totals are not comparable; `profile:session` + reports output-normalised metrics for this reason. +- **Revert what you cannot measure.** A change that does not move its target + metric is unvalidated complexity, not a small win. +- **Reproduction may need production scale you do not have.** A threshold effect + is invisible below its threshold. Compare the reporter's scale against yours + on the dimension the code keys on before concluding a bug is absent. + +## Module Layout + +| File | Responsibility | +|---|---| +| `cdp.mjs` | Chrome launch, target discovery, minimal CDP client. Owns the anti-throttling launch flags. | +| `metrics.mjs` | Metric derivations shared by the profilers: growth rates, percentiles, long-task and trace-event summaries. | +| `cpu-profile.mjs` | Aggregates `Profiler.stop()` output into self time per function. | +| `idle-probe.mjs` | Page-side instrumentation installed before application code runs; attributes scheduled work to the call site that scheduled it. Must never change observable behaviour. | +| `scenario.mjs` | Shared scenario setup, currently sidebar expansion. Setup always runs before the measured window. | +| `animation-fixture.html` | Isolated animation variants for `profile:animation`. | diff --git a/scripts/perf/animation-fixture.html b/scripts/perf/animation-fixture.html new file mode 100644 index 00000000..33cccb9c --- /dev/null +++ b/scripts/perf/animation-fixture.html @@ -0,0 +1,65 @@ + + + + +OpenChamber animation cost fixture + + + +
+ + + diff --git a/scripts/profile-animation.mjs b/scripts/profile-animation.mjs new file mode 100644 index 00000000..b0bdc632 --- /dev/null +++ b/scripts/profile-animation.mjs @@ -0,0 +1,207 @@ +#!/usr/bin/env node +/** + * Measures what a CSS animation costs, in isolation. + * + * A continuously running animation is one of the few things an idle interface + * keeps paying for, and the price depends entirely on whether the compositor + * can drive the animated property. Compositor-driven properties cost a small + * constant; everything else recalculates style on every frame, roughly an order + * of magnitude more. + * + * Rather than rebuilding the application and streaming a response to answer + * that question, this command serves a fixture page and measures each variant + * directly, so a whole comparison takes seconds. Add a variant to + * `perf/animation-fixture.html` to measure a property or technique that is not + * listed yet. + * + * The output answers two questions the fixture is designed for: + * - which properties are cheap to animate; + * - whether cost scales with the number of animated elements (`--count`). + */ + +import { createReadStream } from "node:fs" +import { createServer } from "node:http" +import { homedir } from "node:os" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import process from "node:process" + +import { CdpClient, createPageTarget, launchChrome, reservePort, resolveChrome, wait } from "./perf/cdp.mjs" +import { metricMap, round } from "./perf/metrics.mjs" + +const scriptDirectory = dirname(fileURLToPath(import.meta.url)) +const fixturePath = join(scriptDirectory, "perf", "animation-fixture.html") + +const DEFAULT_VARIANTS = [ + "none", + "transform-rotate", + "transform-rotate-willchange", + "transform-rotate-steps", + "transform-rotate-wrapper", + "rotate-property", + "opacity", + "translate", + "scale", + "background-position", + "border-color", + "box-shadow", + "filter", + "width", +] + +const HELP = `Usage: bun run profile:animation -- [options] + +Measures the idle cost of CSS animations using an isolated fixture page. + +Options: + --variant Measure only this variant (repeatable) + --count Animated elements per variant (default: 2) + --duration Measurement window per variant (default: 10) + --settle Wait before measuring each variant (default: 3) + --chrome Chrome/Chromium executable + --profile-dir Reusable isolated Chrome profile + --headed Show the browser (default: headless) + --json Print results as JSON + --help Show this help + +Variants live in scripts/perf/animation-fixture.html. Add one there to measure +a property or technique that is not covered.` + +const parseArgs = (argv) => { + const options = { + variants: [], + count: 2, + duration: 10, + settle: 3, + chrome: null, + profileDir: join(homedir(), ".openchamber", "browser-profile-google-chrome"), + headless: true, + json: false, + } + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index] + if (value === "--help") return { ...options, help: true } + else if (value === "--headed") options.headless = false + else if (value === "--json") options.json = true + else if (value === "--variant") options.variants.push(argv[++index]) + else if (value === "--count") options.count = Number(argv[++index]) + else if (value === "--duration") options.duration = Number(argv[++index]) + else if (value === "--settle") options.settle = Number(argv[++index]) + else if (value === "--chrome") options.chrome = argv[++index] + else if (value === "--profile-dir") options.profileDir = argv[++index] + else throw new Error(`Unknown option: ${value}`) + } + if (!Number.isFinite(options.duration) || options.duration <= 0) throw new Error("--duration must be a positive number") + if (!Number.isFinite(options.count) || options.count < 1) throw new Error("--count must be at least 1") + if (options.variants.length === 0) options.variants = DEFAULT_VARIANTS + return options +} + +/** Serves only the fixture, on an ephemeral loopback port. */ +const startFixtureServer = () => new Promise((resolvePromise, reject) => { + const server = createServer((_request, response) => { + response.writeHead(200, { "content-type": "text/html; charset=utf-8" }) + createReadStream(fixturePath).pipe(response) + }) + server.on("error", reject) + server.listen(0, "127.0.0.1", () => { + const address = server.address() + if (!address || typeof address === "string") { + server.close() + reject(new Error("Could not bind the fixture server")) + return + } + resolvePromise({ server, port: address.port }) + }) +}) + +const measureVariant = async (client, url, options) => { + const loaded = client.once("Page.loadEventFired", 30_000) + await client.send("Page.navigate", { url }) + await loaded + await wait(options.settle * 1000) + + const before = metricMap((await client.send("Performance.getMetrics")).metrics) + const startedAt = Date.now() + await wait(options.duration * 1000) + const elapsedSeconds = (Date.now() - startedAt) / 1000 + const after = metricMap((await client.send("Performance.getMetrics")).metrics) + + const delta = (name) => Number(after[name] ?? 0) - Number(before[name] ?? 0) + return { + recalcStylePerSecond: round(delta("RecalcStyleCount") / elapsedSeconds), + layoutsPerSecond: round(delta("LayoutCount") / elapsedSeconds), + mainThreadBusyPercent: round((delta("TaskDuration") / elapsedSeconds) * 100), + recalcStyleMsPerSecond: round((delta("RecalcStyleDuration") / elapsedSeconds) * 1000), + } +} + +const main = async () => { + const options = parseArgs(process.argv.slice(2)) + if (options.help) { + console.log(HELP) + return + } + + const chrome = resolveChrome(options.chrome) + const { server, port: fixturePort } = await startFixtureServer() + const debuggingPort = await reservePort() + const chromeProcess = launchChrome({ + chrome, + profileDir: resolve(options.profileDir), + port: debuggingPort, + headless: options.headless, + }) + + let client + const results = [] + try { + const target = await createPageTarget(debuggingPort) + client = new CdpClient(target.webSocketDebuggerUrl) + await client.connect() + await Promise.all([ + client.send("Page.enable"), + client.send("Runtime.enable"), + client.send("Performance.enable"), + ]) + await client.send("Emulation.setDeviceMetricsOverride", { + width: 1200, height: 800, deviceScaleFactor: 1, mobile: false, + }) + + console.log(`Measuring ${options.variants.length} variants, ${options.count} element(s) each, ${options.duration}s per variant.\n`) + for (const variant of options.variants) { + const url = `http://127.0.0.1:${fixturePort}/?variant=${encodeURIComponent(variant)}&count=${options.count}` + const measured = await measureVariant(client, url, options) + results.push({ variant, ...measured }) + if (!options.json) { + console.log( + `${variant.padEnd(30)} recalc/s ${String(measured.recalcStylePerSecond).padStart(8)}` + + ` layout/s ${String(measured.layoutsPerSecond).padStart(6)}` + + ` busy% ${String(measured.mainThreadBusyPercent).padStart(6)}`, + ) + } + } + + if (options.json) { + console.log(JSON.stringify({ count: options.count, durationSeconds: options.duration, results }, null, 2)) + return + } + + const baseline = results.find((entry) => entry.variant === "none") + if (baseline) { + console.log( + `\nA still page costs ${baseline.recalcStylePerSecond} style recalculations per second.` + + " Compositor-driven properties add a small constant; anything far above that recalculates every frame.", + ) + } + } finally { + client?.close() + if (!chromeProcess.killed) chromeProcess.kill("SIGTERM") + server.close() + } +} + +main().catch((error) => { + console.error(`Animation profiling failed: ${error.message}`) + process.exitCode = 1 +})