Files
David Luzar e7b534668a ci(repo): measure each master commit against the one before it (#12484)
GitHub's hosted runners differ from run to run (region, CPU), so a commit's
timings only compare with timings from the same runner.

- The perf job builds the commit and master before the push and loads both
  alternately (`yarn stats:compare --base <commit> --iterations 6`), instead of
  waiting for staging; without staging there's nothing to cancel, so every merge
  gets its run.
- `perf.mjs --base` takes a commit: checked out into a temporary worktree with its
  own dependencies, built and served like this checkout. `--out *.json` writes both
  builds' loads; `--wait-for` is gone. One unrecorded warm-up load per build, and
  the runner's CPU in the environment.
- `post.mjs` sends the commit time (the admin orders runs by it) and the base
  build; perf runs need one.

Claude Opus 5.5 (xhigh)
2026-10-28 16:29:35 +01:00

1140 lines
34 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
/**
* Measures excalidraw.com in a real browser (Playwright, Chromium) for the
* Excalidraw+ admin's Performance page — page loads, and panning and zooming
* a large scene: every scenario `--iterations` times, each in a fresh browser
* context (cold HTTP cache) on a throttled fast-desktop connection — the
* conditions of Excalidraw+'s own runs, so the apps' numbers compare.
*
* `--compare` (`yarn stats:compare`) measures two builds on this machine,
* alternating them, and prints a table of their differences: this checkout's
* production build — built and served locally — or `--url`, against `--base`,
* a url (staging by default) or a commit, checked out and built the same way.
* The machine's speed shows in every timing, so only builds measured side by
* side compare. Master CI compares every commit with the one before it, and
* `post.mjs` stores both builds' loads.
*
* node scripts/stats/perf.mjs --url http://localhost:3000 --iterations 3
* node scripts/stats/perf.mjs --scenario oss-pan-zoom --iterations 1
* yarn stats:compare [--iterations 10] [--out diff.csv]
* yarn stats:compare --base master --out perf.json
* yarn stats:compare --url http://localhost:3000 --base https://excalidraw.com
*/
import { execFileSync, spawn } from "node:child_process";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
import { setTimeout as sleep } from "node:timers/promises";
import { chromium } from "playwright";
const STAGING_URL = "https://staging.excalidraw.com";
const { values: args } = parseArgs({
options: {
// the app to measure — staging by default; with --compare, this checkout's build
url: { type: "string" },
iterations: { type: "string", default: "5" },
// the JSON file to write the run to (printed when left out); with --compare, a
// `.csv` gets the table of differences, a JSON file both builds' loads
out: { type: "string" },
// compare --url (this checkout's build by default) with --base, measured alternately
compare: { type: "boolean", default: false },
// a url, or a commit to build
base: { type: "string", default: STAGING_URL },
// only these scenarios (comma-separated ids)
scenario: { type: "string" },
},
});
const withoutSlash = (url) => url.replace(/\/$/, "");
const APP_URL = withoutSlash(args.url ?? STAGING_URL);
/**
* Loads per scenario (per build with --compare). Consecutive loads differ a
* little (one browser runs them all), so --compare alternates which build
* loads first and needs an even count for both to get the same positions.
*/
const ITERATIONS = args.compare
? Math.ceil(Number(args.iterations) / 2) * 2
: Number(args.iterations);
const REPO_ROOT = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"../..",
);
/** Same as Excalidraw+'s runs (`PERF_ENVIRONMENT` there), close to Lighthouse's desktop preset. */
const PERF_ENVIRONMENT = {
viewport: { width: 1440, height: 900 },
network: { latencyMs: 40, downloadKbps: 10_240, uploadKbps: 10_240 },
};
const LOAD_TIMEOUT_MS = 60_000;
/** Paints right after ready (the LCP is often one) are reported asynchronously. */
const PAINT_SETTLE_MS = 1_000;
/** Between loads, so the browser's work in the background after one (closing it, a renderer for the next) is done. */
const BETWEEN_LOADS_MS = 1_000;
/** Analytics and error reporting stay out of the measured loads (and out of the stats). */
const BLOCKED_URLS = [
"https://*.simpleanalyticscdn.com/**",
"https://*.sentry.io/**",
"https://sentry.io/**",
];
/** The app marks when its entry module runs (`excalidraw-app/index.tsx`), ending the "HTML + JS" phase. */
const APP_START_MARK = "excalidraw:app-start";
/** The app's local storage keys (`STORAGE_KEYS` in `excalidraw-app/app_constants.ts`). */
const STORAGE_KEYS = { elements: "excalidraw", appState: "excalidraw-state" };
const LARGE_SCENE_GRID = {
columns: 50,
rows: 40,
cellWidth: 140,
cellHeight: 110,
};
/**
* The same 2,000 elements every run (seeded): shapes, text, arrows, lines and
* freedraw strokes on a grid, zoomed out so all of them are drawn.
*/
function createLargeScene() {
// mulberry32
let seed = 42;
const random = () => {
seed = (seed + 0x6d2b79f5) | 0;
let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const pick = (list) => list[Math.floor(random() * list.length)];
const words = [
"Idea",
"Draft",
"Review",
"Ship it",
"Backlog",
"Q3 plan",
"API",
"Users",
];
const { columns, rows, cellWidth, cellHeight } = LARGE_SCENE_GRID;
const elements = [];
for (let row = 0; row < rows; row++) {
for (let column = 0; column < columns; column++) {
const base = {
id: `perf-${elements.length}`,
x: column * cellWidth,
y: row * cellHeight,
width: 100,
height: 70,
angle: 0,
strokeColor: pick([
"#1e1e1e",
"#e03131",
"#2f9e44",
"#1971c2",
"#f08c00",
]),
backgroundColor: pick([
"transparent",
"#ffc9c9",
"#b2f2bb",
"#a5d8ff",
"#ffec99",
]),
fillStyle: pick(["solid", "hachure", "cross-hatch"]),
strokeWidth: 2,
strokeStyle: "solid",
roughness: 1,
opacity: 100,
groupIds: [],
frameId: null,
roundness: null,
seed: Math.floor(random() * 2 ** 31),
version: 1,
versionNonce: 0,
isDeleted: false,
boundElements: null,
updated: 1,
link: null,
locked: false,
};
const kind = random();
if (kind < 0.35) {
elements.push({ ...base, type: "rectangle", roundness: { type: 3 } });
} else if (kind < 0.5) {
elements.push({ ...base, type: "ellipse" });
} else if (kind < 0.6) {
elements.push({ ...base, type: "diamond" });
} else if (kind < 0.75) {
const text = pick(words);
elements.push({
...base,
type: "text",
backgroundColor: "transparent",
width: text.length * 11,
height: 25,
text,
originalText: text,
fontSize: 20,
fontFamily: 5,
textAlign: "left",
verticalAlign: "top",
containerId: null,
autoResize: true,
lineHeight: 1.25,
});
} else if (kind < 0.9) {
const isArrow = kind < 0.85;
elements.push({
...base,
type: isArrow ? "arrow" : "line",
backgroundColor: "transparent",
points: [
[0, 0],
[50, random() * 70 - 35],
[100, 70],
],
lastCommittedPoint: null,
startBinding: null,
endBinding: null,
startArrowhead: null,
endArrowhead: isArrow ? "arrow" : null,
elbowed: false,
});
} else {
const points = Array.from({ length: 40 }, (_, index) => [
index * 2.5,
35 + Math.sin(index / 3) * 30 * random(),
]);
elements.push({
...base,
type: "freedraw",
backgroundColor: "transparent",
points,
pressures: [],
simulatePressure: true,
lastCommittedPoint: null,
});
}
}
}
// everything in view: the grid's size times the zoom fits the viewport, centered
const zoom = 0.2;
const { width, height } = PERF_ENVIRONMENT.viewport;
const appState = {
zoom: { value: zoom },
scrollX: (width / zoom - columns * cellWidth) / 2,
scrollY: (height / zoom - rows * cellHeight) / 2,
};
return { elements, appState };
}
/**
* Collects what the Performance API only reports to observers, and counts the
* static canvas's draw calls — the scene is drawn once it drew elements (runs
* before the app's scripts).
*/
function installPageObservers() {
const state = { lcp: 0, cls: 0, longTasks: [], staticDraws: 0 };
window.__perf = state;
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
state.lcp = entry.startTime;
}
}).observe({ type: "largest-contentful-paint", buffered: true });
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
if (!entry.hadRecentInput) {
state.cls += entry.value;
}
}
}).observe({ type: "layout-shift", buffered: true });
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
state.longTasks.push({
start: entry.startTime,
duration: entry.duration,
});
}
}).observe({ type: "longtask", buffered: true });
const context = CanvasRenderingContext2D.prototype;
for (const method of ["drawImage", "fill", "stroke", "fillText"]) {
const draw = context[method];
context[method] = function (...drawArgs) {
if (this.canvas.classList.contains("static")) {
state.staticDraws += 1;
}
return draw.apply(this, drawArgs);
};
}
}
/** Bytes and requests that finished loading, with the connection throttled. */
async function trackNetwork(cdp) {
const resourceTypes = new Map();
const counters = { requests: 0, transferBytes: 0, jsBytes: 0 };
await cdp.send("Network.enable");
await cdp.send("Network.emulateNetworkConditions", {
offline: false,
latency: PERF_ENVIRONMENT.network.latencyMs,
downloadThroughput: (PERF_ENVIRONMENT.network.downloadKbps * 1024) / 8,
uploadThroughput: (PERF_ENVIRONMENT.network.uploadKbps * 1024) / 8,
});
cdp.on("Network.responseReceived", ({ requestId, type }) => {
resourceTypes.set(requestId, type);
});
cdp.on("Network.loadingFinished", ({ requestId, encodedDataLength }) => {
counters.requests += 1;
counters.transferBytes += encodedDataLength;
if (resourceTypes.get(requestId) === "Script") {
counters.jsBytes += encodedDataLength;
}
});
return { snapshot: () => ({ ...counters }) };
}
/** The first frame's `performance.now()` the scenario's `readyAt` (run in the page, every frame) returned. */
async function waitForReady(page, readyAt) {
const handle = await page.waitForFunction(readyAt, undefined, {
timeout: LOAD_TIMEOUT_MS,
});
return handle.jsonValue();
}
/** Paint, layout shift and main-thread metrics of the page until `until` (`performance.now()`). */
function readPageMetrics(page, until) {
return page.evaluate((untilTime) => {
const { __perf: state } = window;
const [navigation] = performance.getEntriesByType("navigation");
const fcp = performance.getEntriesByName("first-contentful-paint")[0]
?.startTime;
// the main thread blocked between first paint and ready, like Lighthouse's TBT until TTI
const tbt = state.longTasks
.filter(({ start }) => start >= (fcp ?? 0) && start < untilTime)
.reduce((sum, { duration }) => sum + Math.max(0, duration - 50), 0);
return {
ttfb: navigation?.responseStart,
fcp,
lcp: state.lcp || undefined,
cls: state.cls,
tbt,
};
}, until);
}
/** Frames slower than this are long animation frames: a visible stutter. */
const LONG_FRAME_MS = 50;
const repeat = (count, step) => Array.from({ length: count }, () => step);
/**
* The wheel events of the pan-and-zoom scenario, one per frame (null: a
* frame without one, so a zoom's sharp redraw after it stops is measured).
* Shift+wheel pans horizontally, ctrl/cmd+shift+wheel vertically and
* ctrl/cmd+wheel zooms around the pointer (`App.wheel.ts`) — whatever input
* device the editor assumes. From the overview at 20 %: pan it, zoom to
* 100 %, pan into elements not drawn at that zoom yet, zoom back out.
*/
const PAN_ZOOM_STEPS = [
...repeat(30, { shiftKey: true, deltaY: 15 }),
...repeat(30, { shiftKey: true, deltaY: -15 }),
...repeat(40, { ctrlKey: true, deltaY: -2 }),
...repeat(20, null),
...repeat(40, { ctrlKey: true, shiftKey: true, deltaY: 20 }),
...repeat(40, { shiftKey: true, deltaY: 20 }),
...repeat(40, { ctrlKey: true, deltaY: 2 }),
...repeat(20, null),
];
/**
* Plays `PAN_ZOOM_STEPS` over the canvas: each frame's main-thread work, the
* time between the frames, and the static canvas's draws. Headless Chromium
* ticks frames at a fixed 60 Hz (the frame-rate flags don't change it), so the
* time between frames only moves in whole frames; a frame's work — from its
* start until a task posted in it runs, after its rendering: the editor
* drawing the previous wheel event, handling the current one, style, layout
* and paint — shows every millisecond.
*/
function panAndZoom(steps) {
const canvas = document.querySelector(
"canvas.excalidraw__canvas.interactive",
);
const { left, top, width, height } = canvas.getBoundingClientRect();
const position = { clientX: left + width / 2, clientY: top + height / 2 };
const nextFrame = () =>
new Promise((resolve) => requestAnimationFrame(resolve));
const afterRendering = () =>
new Promise((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => resolve(performance.now());
channel.port2.postMessage(null);
});
// zooms anchor on the pointer's last position
canvas.dispatchEvent(
new PointerEvent("pointermove", {
bubbles: true,
pointerType: "mouse",
isPrimary: true,
...position,
}),
);
return (async () => {
const drawsBefore = window.__perf.staticDraws;
const work = [];
const intervals = [];
let frameStart = await nextFrame();
for (const step of steps) {
if (step) {
canvas.dispatchEvent(
new WheelEvent("wheel", {
bubbles: true,
cancelable: true,
deltaMode: 0,
...position,
...step,
}),
);
}
work.push((await afterRendering()) - frameStart);
const nextFrameStart = await nextFrame();
intervals.push(nextFrameStart - frameStart);
frameStart = nextFrameStart;
}
return { work, intervals, draws: window.__perf.staticDraws - drawsBefore };
})();
}
/**
* The mean frame work — the interaction's main-thread work spread over its
* frames, so a slower part of it shows however short — its 95th percentile,
* and the long frames.
*/
function getFrameMetrics({ work, intervals }) {
const sorted = [...work].sort((a, b) => a - b);
return {
frameMean: work.reduce((sum, frame) => sum + frame, 0) / work.length,
frameP95:
sorted[Math.min(sorted.length - 1, Math.floor(0.95 * sorted.length))],
longFrames: intervals.filter((interval) => interval > LONG_FRAME_MS).length,
};
}
const LARGE_SCENE = createLargeScene();
/**
* The scenarios, by their ids in the admin's catalogue (`PERF_SCENARIOS` in
* Excalidraw+); `readyAt` returns `performance.now()` once the page is ready.
* One with `interact` measures that once the page is ready and settled,
* instead of the load.
*/
const SCENARIOS = [
{
id: "oss-editor",
// the welcome screen shows once the app loaded an empty scene
readyAt: () =>
document.querySelector(".welcome-screen-center")?.checkVisibility()
? performance.now()
: 0,
},
{
id: "oss-large-scene",
scene: LARGE_SCENE,
readyAt: () => (window.__perf.staticDraws > 0 ? performance.now() : 0),
},
{
id: "oss-pan-zoom",
scene: LARGE_SCENE,
readyAt: () => (window.__perf.staticDraws > 0 ? performance.now() : 0),
interact: async (page) => {
// fonts and lazy chunks loaded, so their redraws don't land in the frames
await page.waitForLoadState("networkidle", { timeout: LOAD_TIMEOUT_MS });
const frames = await page.evaluate(panAndZoom, PAN_ZOOM_STEPS);
if (!frames.draws) {
throw new Error("The scene didn't redraw while panning and zooming");
}
return getFrameMetrics(frames);
},
},
];
async function measureOnce(browser, scenario, appUrl) {
const context = await browser.newContext({
viewport: PERF_ENVIRONMENT.viewport,
// the service worker's precaching would add to the measured downloads
serviceWorkers: "block",
});
try {
for (const url of BLOCKED_URLS) {
await context.route(url, (route) => route.abort());
}
await context.addInitScript(installPageObservers);
if (scenario.scene) {
await context.addInitScript(
({ keys, elements, appState }) => {
localStorage.setItem(keys.elements, elements);
localStorage.setItem(keys.appState, appState);
},
{
keys: STORAGE_KEYS,
elements: JSON.stringify(scenario.scene.elements),
appState: JSON.stringify(scenario.scene.appState),
},
);
}
const page = await context.newPage();
const network = await trackNetwork(await context.newCDPSession(page));
// without waiting for `load`, so the polling sees the editor as soon as it's ready
await page.goto(appUrl, { timeout: LOAD_TIMEOUT_MS, waitUntil: "commit" });
const ready = await waitForReady(page, scenario.readyAt);
const transferred = network.snapshot();
await page.waitForTimeout(PAINT_SETTLE_MS);
const { appStart, buildSha } = await page.evaluate(
(mark) => ({
appStart: performance.getEntriesByName(mark, "mark")[0]?.startTime,
buildSha: window.__EXCALIDRAW_SHA__ ?? null,
}),
APP_START_MARK,
);
const validBuildSha = /^[0-9a-f]{7,40}$/.test(buildSha ?? "")
? buildSha
: null;
if (scenario.interact) {
return {
metrics: await scenario.interact(page),
phases: {},
buildSha: validBuildSha,
};
}
return {
metrics: {
ready,
...(await readPageMetrics(page, ready)),
...transferred,
},
// builds without the mark have no phases
phases:
appStart === undefined
? {}
: { appCode: Math.min(appStart, ready), editor: ready },
buildSha: validBuildSha,
};
} finally {
await context.close();
}
}
/** A free local port, from the OS. */
const getFreePort = () =>
new Promise((resolve, reject) => {
const server = net.createServer();
server.once("error", reject);
server.listen(0, () => {
const { port } = server.address();
server.close(() => resolve(port));
});
});
const git = (...gitArgs) =>
execFileSync("git", gitArgs, { cwd: REPO_ROOT, encoding: "utf8" }).trim();
/** Runs a command with its output kept back, but for the end of it when it fails. */
function runQuietly(command, commandArgs, options, failure) {
try {
execFileSync(command, commandArgs, {
...options,
stdio: "pipe",
maxBuffer: 256 * 1024 * 1024,
});
} catch (error) {
console.error(
String(error.stderr ?? error)
.trim()
.split("\n")
.slice(-20)
.join("\n"),
);
throw new Error(failure);
}
}
/** This checkout: its commit, and whether it has changes on top. */
function getThisCheckout() {
const sha = git("rev-parse", "HEAD");
return {
dir: REPO_ROOT,
sha,
label: `this checkout (${sha.slice(0, 7)}${
git("status", "--porcelain") ? " + uncommitted changes" : ""
})`,
};
}
/**
* `revision` checked out into a temporary worktree with its own dependencies
* (they may differ from this checkout's), removed when the script ends.
*/
function checkOut(revision) {
const sha = git("rev-parse", "--verify", `${revision}^{commit}`);
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "excalidraw-perf-"));
const dir = path.join(tmpDir, "excalidraw");
const label = sha.startsWith(revision)
? sha.slice(0, 7)
: `${revision} (${sha.slice(0, 7)})`;
console.info(`⏱️ Checking out ${label} and installing its dependencies…`);
process.on("exit", () => {
try {
execFileSync("git", ["worktree", "remove", "--force", dir], {
cwd: REPO_ROOT,
stdio: "ignore",
});
} catch {
// the script ended before git created it
}
fs.rmSync(tmpDir, { recursive: true, force: true });
});
git("worktree", "add", "--quiet", "--detach", dir, sha);
runQuietly(
"yarn",
["install", "--frozen-lockfile", "--prefer-offline"],
// no git hooks for the worktree
{ cwd: dir, env: { ...process.env, HUSKY: "0" } },
`Installing the dependencies of ${label} failed`,
);
return { dir, sha, label };
}
/**
* Builds a checkout's app for production — as Vercel does (`build:app`),
* minus linting — and serves it with `vite preview`: its url, and how to stop
* the server.
*/
async function serveLocalBuild({ dir, sha, label }) {
const appDir = path.join(dir, "excalidraw-app");
const vite = path.join(dir, "node_modules/vite/bin/vite.js");
console.info(`⏱️ Building ${label}…`);
runQuietly(
process.execPath,
[vite, "build"],
{
cwd: appDir,
env: {
...process.env,
VITE_APP_GIT_SHA: sha,
VITE_APP_ENABLE_TRACKING: "true",
VITE_APP_ENABLE_ESLINT: "false",
},
},
`Building ${label} failed (\`yarn build:app\` shows why)`,
);
const port = await getFreePort();
const server = spawn(
process.execPath,
[vite, "preview", "--port", String(port), "--strictPort"],
{ cwd: appDir, stdio: "ignore" },
);
// however the script ends
process.on("exit", () => server.kill());
const url = `http://localhost:${port}`;
for (let attempt = 0; ; attempt++) {
const isUp = await fetch(url)
.then((response) => response.ok)
.catch(() => false);
if (isUp) {
return { url, stop: () => server.kill() };
}
if (attempt > 60 || server.exitCode !== null) {
server.kill();
throw new Error("`vite preview` didn't start");
}
await sleep(500);
}
}
// so a stopped run still removes its worktree and servers
for (const signal of ["SIGINT", "SIGTERM"]) {
process.once(signal, () => process.exit(1));
}
/** The metrics the runner records, in the admin's order (`PERF_METRICS` in Excalidraw+), and their units. */
const METRIC_UNITS = {
ready: "ms",
ttfb: "ms",
fcp: "ms",
lcp: "ms",
tbt: "ms",
cls: "score",
jsBytes: "bytes",
transferBytes: "bytes",
requests: "count",
frameMean: "ms",
frameP95: "ms",
longFrames: "count",
};
/** Linear-interpolated quantile of ascending `sorted` (the admin's too). */
const quantile = (sorted, q) => {
const position = (sorted.length - 1) * q;
const lower = Math.floor(position);
const upper = Math.ceil(position);
return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower);
};
const summarize = (values) => {
const sorted = [...values].sort((a, b) => a - b);
return {
median: quantile(sorted, 0.5),
p25: quantile(sorted, 0.25),
p75: quantile(sorted, 0.75),
};
};
/** Below it, a difference in timings is unlikely to be the loads' noise. */
const SIGNIFICANCE = 0.05;
/**
* The two-sided p-value of the Mann–Whitney U test: how likely the two
* samples' ranks are this far apart if the builds didn't differ — no
* assumption on how timings spread, fit for a few loads per side. Exact (by
* counting the rank arrangements); ties get mid-ranks.
*/
function mannWhitneyP(a, b) {
const ranked = [
...a.map((value) => ({ value, inA: true })),
...b.map((value) => ({ value, inA: false })),
].sort((x, y) => x.value - y.value);
let rankSumA = 0;
for (let i = 0; i < ranked.length; ) {
let j = i;
while (j + 1 < ranked.length && ranked[j + 1].value === ranked[i].value) {
j++;
}
// ranks i + 1 … j + 1, shared
const rank = (i + j + 2) / 2;
for (let k = i; k <= j; k++) {
if (ranked[k].inA) {
rankSumA += rank;
}
}
i = j + 1;
}
const m = a.length;
const n = b.length;
const uA = rankSumA - (m * (m + 1)) / 2;
const u = Math.floor(Math.min(uA, m * n - uA));
// arrangements of m and n values with the statistic `value`
const memo = new Map();
const count = (value, m, n) => {
if (value < 0) {
return 0;
}
if (!m || !n) {
return value === 0 ? 1 : 0;
}
const key = `${value},${m},${n}`;
if (!memo.has(key)) {
memo.set(key, count(value - n, m - 1, n) + count(value, m, n - 1));
}
return memo.get(key);
};
let atMost = 0;
for (let value = 0; value <= u; value++) {
atMost += count(value, m, n);
}
let arrangements = 1;
for (let i = 1; i <= m; i++) {
arrangements = (arrangements * (n + i)) / i;
}
return Math.min(1, (2 * atMost) / arrangements);
}
/**
* Every metric is better lower. Timings differ when the Mann–Whitney test
* says so (`SIGNIFICANCE`); bytes and requests don't vary from load to load,
* so any difference in them counts.
*/
function getVerdict(baseValues, nextValues) {
const isConstant = (values) => values.every((value) => value === values[0]);
const direction = Math.sign(median(nextValues) - median(baseValues));
const p =
isConstant(baseValues) && isConstant(nextValues)
? direction
? 0
: 1
: mannWhitneyP(baseValues, nextValues);
return {
p,
verdict:
p < SIGNIFICANCE && direction
? direction > 0
? "worse"
: "better"
: "~",
};
}
const median = (values) =>
quantile(
[...values].sort((a, b) => a - b),
0.5,
);
/** One row per scenario and metric both targets measured. */
function compareTargets(scenarios, base, next) {
const rows = [];
for (const scenario of scenarios) {
const failed = (target) =>
target.samples.filter((s) => s.scenario === scenario.id && s.error)
.length;
for (const [metric, unit] of Object.entries(METRIC_UNITS)) {
const values = (target) =>
target.samples.flatMap((sample) =>
sample.scenario === scenario.id &&
!sample.error &&
sample.metrics[metric] !== undefined
? [sample.metrics[metric]]
: [],
);
const baseValues = values(base);
const nextValues = values(next);
if (!baseValues.length || !nextValues.length) {
continue;
}
const baseStats = summarize(baseValues);
const nextStats = summarize(nextValues);
const delta = nextStats.median - baseStats.median;
rows.push({
scenario: scenario.id,
metric,
unit,
base: baseStats,
next: nextStats,
delta,
deltaPercent: baseStats.median
? (delta / baseStats.median) * 100
: null,
...getVerdict(baseValues, nextValues),
baseFailed: failed(base),
nextFailed: failed(next),
});
}
}
return rows;
}
const formatValue = (value, unit) =>
unit === "ms"
? `${
value < 100 ? value.toFixed(1) : Math.round(value).toLocaleString("en")
} ms`
: unit === "bytes"
? `${(value / 1024).toLocaleString("en", {
minimumFractionDigits: 1,
maximumFractionDigits: 1,
})} KiB`
: unit === "score"
? value.toFixed(3)
: String(Math.round(value * 10) / 10);
const formatSigned = (value, format) =>
`${value > 0 ? "+" : value < 0 ? "−" : "±"}${format(Math.abs(value))}`;
const VERDICT_COLORS = { worse: 31, better: 32 };
/** The comparison as an aligned table, the verdicts colored on a terminal. */
function printComparison(rows) {
const color = (text, verdict) =>
process.stdout.isTTY && VERDICT_COLORS[verdict]
? `\x1b[${VERDICT_COLORS[verdict]}m${text}\x1b[0m`
: text;
const header = [
"Scenario",
"Metric",
"Base",
"New",
"Change",
"Change %",
"Base p25–p75",
"New p25–p75",
"p",
"",
];
const lines = rows.map((row, index) => [
index && rows[index - 1].scenario === row.scenario ? "" : row.scenario,
row.metric,
formatValue(row.base.median, row.unit),
formatValue(row.next.median, row.unit),
formatSigned(row.delta, (value) => formatValue(value, row.unit)),
row.deltaPercent === null
? "—"
: formatSigned(row.deltaPercent, (value) => `${value.toFixed(1)} %`),
`${formatValue(row.base.p25, row.unit)} – ${formatValue(
row.base.p75,
row.unit,
)}`,
`${formatValue(row.next.p25, row.unit)} – ${formatValue(
row.next.p75,
row.unit,
)}`,
row.p < 0.001 ? "<0.001" : row.p.toFixed(3),
row.verdict,
]);
const widths = header.map((title, column) =>
Math.max(title.length, ...lines.map((line) => line[column].length)),
);
// the names left-aligned, the numbers right-aligned
const pad = (cell, column) =>
column < 2 || column === header.length - 1
? cell.padEnd(widths[column])
: cell.padStart(widths[column]);
console.info(header.map(pad).join(" "));
console.info(widths.map((width) => "─".repeat(width)).join(" "));
rows.forEach((row, index) =>
console.info(color(lines[index].map(pad).join(" "), row.verdict)),
);
}
const CSV_COLUMNS = [
"scenario",
"metric",
"unit",
"base_median",
"base_p25",
"base_p75",
"new_median",
"new_p25",
"new_p75",
"change",
"change_percent",
"p_value",
"verdict",
"base_failed",
"new_failed",
];
const toCsv = (rows) =>
[
CSV_COLUMNS.join(","),
...rows.map((row) =>
[
row.scenario,
row.metric,
row.unit,
row.base.median,
row.base.p25,
row.base.p75,
row.next.median,
row.next.p25,
row.next.p75,
row.delta,
row.deltaPercent ?? "",
row.p,
row.verdict,
row.baseFailed,
row.nextFailed,
].join(","),
),
].join("\n");
const scenarios = args.scenario
? SCENARIOS.filter(({ id }) => args.scenario.split(",").includes(id))
: SCENARIOS;
/** A build to measure: its url, its commit when it's built here, and its loads. */
const target = (name, { url, label = url, commitSha = null }) => ({
name,
url,
label,
commitSha,
samples: [],
buildSha: null,
});
const stopServers = [];
/** A checkout's build, served here. */
async function localTarget(name, checkout) {
const { url, stop } = await serveLocalBuild(checkout);
stopServers.push(stop);
return target(name, {
url,
label: `${checkout.label} at ${url}`,
commitSha: checkout.sha,
});
}
const isUrl = (value) => /^https?:\/\//.test(value);
// with --compare, the base and the new build alternate: whichever loads first
// switches every iteration, so the machine drifting (warming up, other work)
// weighs on both alike
const targets = args.compare
? [
isUrl(args.base)
? target("base", { url: withoutSlash(args.base) })
: await localTarget("base", checkOut(args.base)),
args.url
? target("new", { url: APP_URL })
: await localTarget("new", getThisCheckout()),
]
: [target(null, { url: APP_URL })];
console.info(
args.compare
? `⏱️ Comparing ${targets[1].label} with ${targets[0].label}, ${ITERATIONS} loads each per scenario, alternating…`
: `⏱️ Measuring ${APP_URL}, ${ITERATIONS} loads per scenario…`,
);
const browser = await chromium.launch();
const browserVersion = browser.version();
try {
// the browser's first load is slower than the next ones: one of each build, not recorded
for (const target of targets) {
await measureOnce(browser, scenarios[0], target.url).catch(() => {});
}
for (const scenario of scenarios) {
for (let iteration = 1; iteration <= ITERATIONS; iteration++) {
const order = iteration % 2 ? targets : [...targets].reverse();
for (const target of order) {
const label = `${scenario.id}${
target.name ? ` ${target.name}` : ""
} #${iteration}`;
try {
await sleep(BETWEEN_LOADS_MS);
const load = await measureOnce(browser, scenario, target.url);
target.buildSha ??= load.buildSha;
target.samples.push({
scenario: scenario.id,
iteration,
metrics: load.metrics,
phases: load.phases,
error: null,
});
console.info(
`⏱️ ${label}: ${
scenario.interact
? `mean frame ${Math.round(
load.metrics.frameMean,
)} ms, p95 ${Math.round(load.metrics.frameP95)} ms, ${
load.metrics.longFrames
} long`
: `ready in ${Math.round(load.metrics.ready)} ms`
}`,
);
} catch (error) {
// a failed load is a sample with its error, so one flaky load doesn't lose the run
const message = (
error instanceof Error ? error.message : String(error)
).split("\n")[0];
target.samples.push({
scenario: scenario.id,
iteration,
metrics: {},
phases: {},
error: message,
});
console.warn(`⏱️ ${label} failed: ${message}`);
}
}
}
}
} finally {
await browser.close();
stopServers.forEach((stop) => stop());
}
if (targets.some(({ samples }) => !samples.some(({ error }) => !error))) {
console.error("⏱️ Every page load failed");
process.exit(1);
}
const [base, next] = args.compare ? targets : [null, targets[0]];
// with --compare, the base build's loads too (`post.mjs` stores them with the run)
const run = {
appUrl: next.url,
buildSha: next.buildSha,
environment: {
...PERF_ENVIRONMENT,
runner: process.env.RUNNER_NAME ?? null,
cpu: os.cpus()[0]?.model ?? null,
browser: `chromium ${browserVersion}`,
iterations: ITERATIONS,
},
samples: next.samples,
...(base && {
base: {
// a url's: the build it served
commitSha: base.commitSha ?? base.buildSha,
buildSha: base.buildSha,
samples: base.samples,
},
}),
};
if (args.compare) {
const rows = compareTargets(scenarios, base, next);
const describe = ({ label, buildSha }) =>
`${label}${buildSha ? ` (build ${buildSha.slice(0, 7)})` : ""}`;
console.info(`\nBase: ${describe(base)}\nNew: ${describe(next)}\n`);
printComparison(rows);
for (const row of rows.filter(
(row, index) =>
(row.baseFailed || row.nextFailed) &&
rows.findIndex(({ scenario }) => scenario === row.scenario) === index,
)) {
console.warn(
`⏱️ ${row.scenario}: ${row.baseFailed} base and ${row.nextFailed} new loads failed`,
);
}
if (args.out) {
fs.writeFileSync(
args.out,
args.out.endsWith(".csv")
? `${toCsv(rows)}\n`
: JSON.stringify(run, null, 2),
);
console.info(`\nWrote ${args.out}`);
}
} else if (args.out) {
fs.writeFileSync(args.out, JSON.stringify(run, null, 2));
} else {
console.info(JSON.stringify(run, null, 2));
}