mirror of
https://github.com/excalidraw/excalidraw.git
synced 2026-11-20 21:23:22 +03:00
GitHub's hosted runners differ from run to run (region, CPU), so a commit's timings only compare with timings from the same runner. - The perf job builds the commit and master before the push and loads both alternately (`yarn stats:compare --base <commit> --iterations 6`), instead of waiting for staging; without staging there's nothing to cancel, so every merge gets its run. - `perf.mjs --base` takes a commit: checked out into a temporary worktree with its own dependencies, built and served like this checkout. `--out *.json` writes both builds' loads; `--wait-for` is gone. One unrecorded warm-up load per build, and the runner's CPU in the environment. - `post.mjs` sends the commit time (the admin orders runs by it) and the base build; perf runs need one. Claude Opus 5.5 (xhigh)
1140 lines
34 KiB
JavaScript
1140 lines
34 KiB
JavaScript
#!/usr/bin/env node
|
||
|
||
/**
|
||
* Measures excalidraw.com in a real browser (Playwright, Chromium) for the
|
||
* Excalidraw+ admin's Performance page — page loads, and panning and zooming
|
||
* a large scene: every scenario `--iterations` times, each in a fresh browser
|
||
* context (cold HTTP cache) on a throttled fast-desktop connection — the
|
||
* conditions of Excalidraw+'s own runs, so the apps' numbers compare.
|
||
*
|
||
* `--compare` (`yarn stats:compare`) measures two builds on this machine,
|
||
* alternating them, and prints a table of their differences: this checkout's
|
||
* production build — built and served locally — or `--url`, against `--base`,
|
||
* a url (staging by default) or a commit, checked out and built the same way.
|
||
* The machine's speed shows in every timing, so only builds measured side by
|
||
* side compare. Master CI compares every commit with the one before it, and
|
||
* `post.mjs` stores both builds' loads.
|
||
*
|
||
* node scripts/stats/perf.mjs --url http://localhost:3000 --iterations 3
|
||
* node scripts/stats/perf.mjs --scenario oss-pan-zoom --iterations 1
|
||
* yarn stats:compare [--iterations 10] [--out diff.csv]
|
||
* yarn stats:compare --base master --out perf.json
|
||
* yarn stats:compare --url http://localhost:3000 --base https://excalidraw.com
|
||
*/
|
||
|
||
import { execFileSync, spawn } from "node:child_process";
|
||
import fs from "node:fs";
|
||
import net from "node:net";
|
||
import os from "node:os";
|
||
import path from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
import { parseArgs } from "node:util";
|
||
import { setTimeout as sleep } from "node:timers/promises";
|
||
|
||
import { chromium } from "playwright";
|
||
|
||
const STAGING_URL = "https://staging.excalidraw.com";
|
||
|
||
const { values: args } = parseArgs({
|
||
options: {
|
||
// the app to measure — staging by default; with --compare, this checkout's build
|
||
url: { type: "string" },
|
||
iterations: { type: "string", default: "5" },
|
||
// the JSON file to write the run to (printed when left out); with --compare, a
|
||
// `.csv` gets the table of differences, a JSON file both builds' loads
|
||
out: { type: "string" },
|
||
// compare --url (this checkout's build by default) with --base, measured alternately
|
||
compare: { type: "boolean", default: false },
|
||
// a url, or a commit to build
|
||
base: { type: "string", default: STAGING_URL },
|
||
// only these scenarios (comma-separated ids)
|
||
scenario: { type: "string" },
|
||
},
|
||
});
|
||
|
||
const withoutSlash = (url) => url.replace(/\/$/, "");
|
||
|
||
const APP_URL = withoutSlash(args.url ?? STAGING_URL);
|
||
/**
|
||
* Loads per scenario (per build with --compare). Consecutive loads differ a
|
||
* little (one browser runs them all), so --compare alternates which build
|
||
* loads first and needs an even count for both to get the same positions.
|
||
*/
|
||
const ITERATIONS = args.compare
|
||
? Math.ceil(Number(args.iterations) / 2) * 2
|
||
: Number(args.iterations);
|
||
|
||
const REPO_ROOT = path.resolve(
|
||
path.dirname(fileURLToPath(import.meta.url)),
|
||
"../..",
|
||
);
|
||
|
||
/** Same as Excalidraw+'s runs (`PERF_ENVIRONMENT` there), close to Lighthouse's desktop preset. */
|
||
const PERF_ENVIRONMENT = {
|
||
viewport: { width: 1440, height: 900 },
|
||
network: { latencyMs: 40, downloadKbps: 10_240, uploadKbps: 10_240 },
|
||
};
|
||
|
||
const LOAD_TIMEOUT_MS = 60_000;
|
||
|
||
/** Paints right after ready (the LCP is often one) are reported asynchronously. */
|
||
const PAINT_SETTLE_MS = 1_000;
|
||
|
||
/** Between loads, so the browser's work in the background after one (closing it, a renderer for the next) is done. */
|
||
const BETWEEN_LOADS_MS = 1_000;
|
||
|
||
/** Analytics and error reporting stay out of the measured loads (and out of the stats). */
|
||
const BLOCKED_URLS = [
|
||
"https://*.simpleanalyticscdn.com/**",
|
||
"https://*.sentry.io/**",
|
||
"https://sentry.io/**",
|
||
];
|
||
|
||
/** The app marks when its entry module runs (`excalidraw-app/index.tsx`), ending the "HTML + JS" phase. */
|
||
const APP_START_MARK = "excalidraw:app-start";
|
||
|
||
/** The app's local storage keys (`STORAGE_KEYS` in `excalidraw-app/app_constants.ts`). */
|
||
const STORAGE_KEYS = { elements: "excalidraw", appState: "excalidraw-state" };
|
||
|
||
const LARGE_SCENE_GRID = {
|
||
columns: 50,
|
||
rows: 40,
|
||
cellWidth: 140,
|
||
cellHeight: 110,
|
||
};
|
||
|
||
/**
|
||
* The same 2,000 elements every run (seeded): shapes, text, arrows, lines and
|
||
* freedraw strokes on a grid, zoomed out so all of them are drawn.
|
||
*/
|
||
function createLargeScene() {
|
||
// mulberry32
|
||
let seed = 42;
|
||
const random = () => {
|
||
seed = (seed + 0x6d2b79f5) | 0;
|
||
let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
|
||
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||
};
|
||
const pick = (list) => list[Math.floor(random() * list.length)];
|
||
const words = [
|
||
"Idea",
|
||
"Draft",
|
||
"Review",
|
||
"Ship it",
|
||
"Backlog",
|
||
"Q3 plan",
|
||
"API",
|
||
"Users",
|
||
];
|
||
const { columns, rows, cellWidth, cellHeight } = LARGE_SCENE_GRID;
|
||
const elements = [];
|
||
|
||
for (let row = 0; row < rows; row++) {
|
||
for (let column = 0; column < columns; column++) {
|
||
const base = {
|
||
id: `perf-${elements.length}`,
|
||
x: column * cellWidth,
|
||
y: row * cellHeight,
|
||
width: 100,
|
||
height: 70,
|
||
angle: 0,
|
||
strokeColor: pick([
|
||
"#1e1e1e",
|
||
"#e03131",
|
||
"#2f9e44",
|
||
"#1971c2",
|
||
"#f08c00",
|
||
]),
|
||
backgroundColor: pick([
|
||
"transparent",
|
||
"#ffc9c9",
|
||
"#b2f2bb",
|
||
"#a5d8ff",
|
||
"#ffec99",
|
||
]),
|
||
fillStyle: pick(["solid", "hachure", "cross-hatch"]),
|
||
strokeWidth: 2,
|
||
strokeStyle: "solid",
|
||
roughness: 1,
|
||
opacity: 100,
|
||
groupIds: [],
|
||
frameId: null,
|
||
roundness: null,
|
||
seed: Math.floor(random() * 2 ** 31),
|
||
version: 1,
|
||
versionNonce: 0,
|
||
isDeleted: false,
|
||
boundElements: null,
|
||
updated: 1,
|
||
link: null,
|
||
locked: false,
|
||
};
|
||
const kind = random();
|
||
|
||
if (kind < 0.35) {
|
||
elements.push({ ...base, type: "rectangle", roundness: { type: 3 } });
|
||
} else if (kind < 0.5) {
|
||
elements.push({ ...base, type: "ellipse" });
|
||
} else if (kind < 0.6) {
|
||
elements.push({ ...base, type: "diamond" });
|
||
} else if (kind < 0.75) {
|
||
const text = pick(words);
|
||
elements.push({
|
||
...base,
|
||
type: "text",
|
||
backgroundColor: "transparent",
|
||
width: text.length * 11,
|
||
height: 25,
|
||
text,
|
||
originalText: text,
|
||
fontSize: 20,
|
||
fontFamily: 5,
|
||
textAlign: "left",
|
||
verticalAlign: "top",
|
||
containerId: null,
|
||
autoResize: true,
|
||
lineHeight: 1.25,
|
||
});
|
||
} else if (kind < 0.9) {
|
||
const isArrow = kind < 0.85;
|
||
elements.push({
|
||
...base,
|
||
type: isArrow ? "arrow" : "line",
|
||
backgroundColor: "transparent",
|
||
points: [
|
||
[0, 0],
|
||
[50, random() * 70 - 35],
|
||
[100, 70],
|
||
],
|
||
lastCommittedPoint: null,
|
||
startBinding: null,
|
||
endBinding: null,
|
||
startArrowhead: null,
|
||
endArrowhead: isArrow ? "arrow" : null,
|
||
elbowed: false,
|
||
});
|
||
} else {
|
||
const points = Array.from({ length: 40 }, (_, index) => [
|
||
index * 2.5,
|
||
35 + Math.sin(index / 3) * 30 * random(),
|
||
]);
|
||
elements.push({
|
||
...base,
|
||
type: "freedraw",
|
||
backgroundColor: "transparent",
|
||
points,
|
||
pressures: [],
|
||
simulatePressure: true,
|
||
lastCommittedPoint: null,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
// everything in view: the grid's size times the zoom fits the viewport, centered
|
||
const zoom = 0.2;
|
||
const { width, height } = PERF_ENVIRONMENT.viewport;
|
||
const appState = {
|
||
zoom: { value: zoom },
|
||
scrollX: (width / zoom - columns * cellWidth) / 2,
|
||
scrollY: (height / zoom - rows * cellHeight) / 2,
|
||
};
|
||
|
||
return { elements, appState };
|
||
}
|
||
|
||
/**
|
||
* Collects what the Performance API only reports to observers, and counts the
|
||
* static canvas's draw calls — the scene is drawn once it drew elements (runs
|
||
* before the app's scripts).
|
||
*/
|
||
function installPageObservers() {
|
||
const state = { lcp: 0, cls: 0, longTasks: [], staticDraws: 0 };
|
||
window.__perf = state;
|
||
|
||
new PerformanceObserver((list) => {
|
||
for (const entry of list.getEntries()) {
|
||
state.lcp = entry.startTime;
|
||
}
|
||
}).observe({ type: "largest-contentful-paint", buffered: true });
|
||
|
||
new PerformanceObserver((list) => {
|
||
for (const entry of list.getEntries()) {
|
||
if (!entry.hadRecentInput) {
|
||
state.cls += entry.value;
|
||
}
|
||
}
|
||
}).observe({ type: "layout-shift", buffered: true });
|
||
|
||
new PerformanceObserver((list) => {
|
||
for (const entry of list.getEntries()) {
|
||
state.longTasks.push({
|
||
start: entry.startTime,
|
||
duration: entry.duration,
|
||
});
|
||
}
|
||
}).observe({ type: "longtask", buffered: true });
|
||
|
||
const context = CanvasRenderingContext2D.prototype;
|
||
for (const method of ["drawImage", "fill", "stroke", "fillText"]) {
|
||
const draw = context[method];
|
||
context[method] = function (...drawArgs) {
|
||
if (this.canvas.classList.contains("static")) {
|
||
state.staticDraws += 1;
|
||
}
|
||
return draw.apply(this, drawArgs);
|
||
};
|
||
}
|
||
}
|
||
|
||
/** Bytes and requests that finished loading, with the connection throttled. */
|
||
async function trackNetwork(cdp) {
|
||
const resourceTypes = new Map();
|
||
const counters = { requests: 0, transferBytes: 0, jsBytes: 0 };
|
||
|
||
await cdp.send("Network.enable");
|
||
await cdp.send("Network.emulateNetworkConditions", {
|
||
offline: false,
|
||
latency: PERF_ENVIRONMENT.network.latencyMs,
|
||
downloadThroughput: (PERF_ENVIRONMENT.network.downloadKbps * 1024) / 8,
|
||
uploadThroughput: (PERF_ENVIRONMENT.network.uploadKbps * 1024) / 8,
|
||
});
|
||
|
||
cdp.on("Network.responseReceived", ({ requestId, type }) => {
|
||
resourceTypes.set(requestId, type);
|
||
});
|
||
cdp.on("Network.loadingFinished", ({ requestId, encodedDataLength }) => {
|
||
counters.requests += 1;
|
||
counters.transferBytes += encodedDataLength;
|
||
if (resourceTypes.get(requestId) === "Script") {
|
||
counters.jsBytes += encodedDataLength;
|
||
}
|
||
});
|
||
|
||
return { snapshot: () => ({ ...counters }) };
|
||
}
|
||
|
||
/** The first frame's `performance.now()` the scenario's `readyAt` (run in the page, every frame) returned. */
|
||
async function waitForReady(page, readyAt) {
|
||
const handle = await page.waitForFunction(readyAt, undefined, {
|
||
timeout: LOAD_TIMEOUT_MS,
|
||
});
|
||
return handle.jsonValue();
|
||
}
|
||
|
||
/** Paint, layout shift and main-thread metrics of the page until `until` (`performance.now()`). */
|
||
function readPageMetrics(page, until) {
|
||
return page.evaluate((untilTime) => {
|
||
const { __perf: state } = window;
|
||
const [navigation] = performance.getEntriesByType("navigation");
|
||
const fcp = performance.getEntriesByName("first-contentful-paint")[0]
|
||
?.startTime;
|
||
// the main thread blocked between first paint and ready, like Lighthouse's TBT until TTI
|
||
const tbt = state.longTasks
|
||
.filter(({ start }) => start >= (fcp ?? 0) && start < untilTime)
|
||
.reduce((sum, { duration }) => sum + Math.max(0, duration - 50), 0);
|
||
|
||
return {
|
||
ttfb: navigation?.responseStart,
|
||
fcp,
|
||
lcp: state.lcp || undefined,
|
||
cls: state.cls,
|
||
tbt,
|
||
};
|
||
}, until);
|
||
}
|
||
|
||
/** Frames slower than this are long animation frames: a visible stutter. */
|
||
const LONG_FRAME_MS = 50;
|
||
|
||
const repeat = (count, step) => Array.from({ length: count }, () => step);
|
||
|
||
/**
|
||
* The wheel events of the pan-and-zoom scenario, one per frame (null: a
|
||
* frame without one, so a zoom's sharp redraw after it stops is measured).
|
||
* Shift+wheel pans horizontally, ctrl/cmd+shift+wheel vertically and
|
||
* ctrl/cmd+wheel zooms around the pointer (`App.wheel.ts`) — whatever input
|
||
* device the editor assumes. From the overview at 20 %: pan it, zoom to
|
||
* 100 %, pan into elements not drawn at that zoom yet, zoom back out.
|
||
*/
|
||
const PAN_ZOOM_STEPS = [
|
||
...repeat(30, { shiftKey: true, deltaY: 15 }),
|
||
...repeat(30, { shiftKey: true, deltaY: -15 }),
|
||
...repeat(40, { ctrlKey: true, deltaY: -2 }),
|
||
...repeat(20, null),
|
||
...repeat(40, { ctrlKey: true, shiftKey: true, deltaY: 20 }),
|
||
...repeat(40, { shiftKey: true, deltaY: 20 }),
|
||
...repeat(40, { ctrlKey: true, deltaY: 2 }),
|
||
...repeat(20, null),
|
||
];
|
||
|
||
/**
|
||
* Plays `PAN_ZOOM_STEPS` over the canvas: each frame's main-thread work, the
|
||
* time between the frames, and the static canvas's draws. Headless Chromium
|
||
* ticks frames at a fixed 60 Hz (the frame-rate flags don't change it), so the
|
||
* time between frames only moves in whole frames; a frame's work — from its
|
||
* start until a task posted in it runs, after its rendering: the editor
|
||
* drawing the previous wheel event, handling the current one, style, layout
|
||
* and paint — shows every millisecond.
|
||
*/
|
||
function panAndZoom(steps) {
|
||
const canvas = document.querySelector(
|
||
"canvas.excalidraw__canvas.interactive",
|
||
);
|
||
const { left, top, width, height } = canvas.getBoundingClientRect();
|
||
const position = { clientX: left + width / 2, clientY: top + height / 2 };
|
||
const nextFrame = () =>
|
||
new Promise((resolve) => requestAnimationFrame(resolve));
|
||
const afterRendering = () =>
|
||
new Promise((resolve) => {
|
||
const channel = new MessageChannel();
|
||
channel.port1.onmessage = () => resolve(performance.now());
|
||
channel.port2.postMessage(null);
|
||
});
|
||
|
||
// zooms anchor on the pointer's last position
|
||
canvas.dispatchEvent(
|
||
new PointerEvent("pointermove", {
|
||
bubbles: true,
|
||
pointerType: "mouse",
|
||
isPrimary: true,
|
||
...position,
|
||
}),
|
||
);
|
||
|
||
return (async () => {
|
||
const drawsBefore = window.__perf.staticDraws;
|
||
const work = [];
|
||
const intervals = [];
|
||
let frameStart = await nextFrame();
|
||
for (const step of steps) {
|
||
if (step) {
|
||
canvas.dispatchEvent(
|
||
new WheelEvent("wheel", {
|
||
bubbles: true,
|
||
cancelable: true,
|
||
deltaMode: 0,
|
||
...position,
|
||
...step,
|
||
}),
|
||
);
|
||
}
|
||
work.push((await afterRendering()) - frameStart);
|
||
const nextFrameStart = await nextFrame();
|
||
intervals.push(nextFrameStart - frameStart);
|
||
frameStart = nextFrameStart;
|
||
}
|
||
return { work, intervals, draws: window.__perf.staticDraws - drawsBefore };
|
||
})();
|
||
}
|
||
|
||
/**
|
||
* The mean frame work — the interaction's main-thread work spread over its
|
||
* frames, so a slower part of it shows however short — its 95th percentile,
|
||
* and the long frames.
|
||
*/
|
||
function getFrameMetrics({ work, intervals }) {
|
||
const sorted = [...work].sort((a, b) => a - b);
|
||
return {
|
||
frameMean: work.reduce((sum, frame) => sum + frame, 0) / work.length,
|
||
frameP95:
|
||
sorted[Math.min(sorted.length - 1, Math.floor(0.95 * sorted.length))],
|
||
longFrames: intervals.filter((interval) => interval > LONG_FRAME_MS).length,
|
||
};
|
||
}
|
||
|
||
const LARGE_SCENE = createLargeScene();
|
||
|
||
/**
|
||
* The scenarios, by their ids in the admin's catalogue (`PERF_SCENARIOS` in
|
||
* Excalidraw+); `readyAt` returns `performance.now()` once the page is ready.
|
||
* One with `interact` measures that once the page is ready and settled,
|
||
* instead of the load.
|
||
*/
|
||
const SCENARIOS = [
|
||
{
|
||
id: "oss-editor",
|
||
// the welcome screen shows once the app loaded an empty scene
|
||
readyAt: () =>
|
||
document.querySelector(".welcome-screen-center")?.checkVisibility()
|
||
? performance.now()
|
||
: 0,
|
||
},
|
||
{
|
||
id: "oss-large-scene",
|
||
scene: LARGE_SCENE,
|
||
readyAt: () => (window.__perf.staticDraws > 0 ? performance.now() : 0),
|
||
},
|
||
{
|
||
id: "oss-pan-zoom",
|
||
scene: LARGE_SCENE,
|
||
readyAt: () => (window.__perf.staticDraws > 0 ? performance.now() : 0),
|
||
interact: async (page) => {
|
||
// fonts and lazy chunks loaded, so their redraws don't land in the frames
|
||
await page.waitForLoadState("networkidle", { timeout: LOAD_TIMEOUT_MS });
|
||
const frames = await page.evaluate(panAndZoom, PAN_ZOOM_STEPS);
|
||
if (!frames.draws) {
|
||
throw new Error("The scene didn't redraw while panning and zooming");
|
||
}
|
||
return getFrameMetrics(frames);
|
||
},
|
||
},
|
||
];
|
||
|
||
async function measureOnce(browser, scenario, appUrl) {
|
||
const context = await browser.newContext({
|
||
viewport: PERF_ENVIRONMENT.viewport,
|
||
// the service worker's precaching would add to the measured downloads
|
||
serviceWorkers: "block",
|
||
});
|
||
|
||
try {
|
||
for (const url of BLOCKED_URLS) {
|
||
await context.route(url, (route) => route.abort());
|
||
}
|
||
await context.addInitScript(installPageObservers);
|
||
if (scenario.scene) {
|
||
await context.addInitScript(
|
||
({ keys, elements, appState }) => {
|
||
localStorage.setItem(keys.elements, elements);
|
||
localStorage.setItem(keys.appState, appState);
|
||
},
|
||
{
|
||
keys: STORAGE_KEYS,
|
||
elements: JSON.stringify(scenario.scene.elements),
|
||
appState: JSON.stringify(scenario.scene.appState),
|
||
},
|
||
);
|
||
}
|
||
|
||
const page = await context.newPage();
|
||
const network = await trackNetwork(await context.newCDPSession(page));
|
||
|
||
// without waiting for `load`, so the polling sees the editor as soon as it's ready
|
||
await page.goto(appUrl, { timeout: LOAD_TIMEOUT_MS, waitUntil: "commit" });
|
||
const ready = await waitForReady(page, scenario.readyAt);
|
||
const transferred = network.snapshot();
|
||
await page.waitForTimeout(PAINT_SETTLE_MS);
|
||
const { appStart, buildSha } = await page.evaluate(
|
||
(mark) => ({
|
||
appStart: performance.getEntriesByName(mark, "mark")[0]?.startTime,
|
||
buildSha: window.__EXCALIDRAW_SHA__ ?? null,
|
||
}),
|
||
APP_START_MARK,
|
||
);
|
||
const validBuildSha = /^[0-9a-f]{7,40}$/.test(buildSha ?? "")
|
||
? buildSha
|
||
: null;
|
||
|
||
if (scenario.interact) {
|
||
return {
|
||
metrics: await scenario.interact(page),
|
||
phases: {},
|
||
buildSha: validBuildSha,
|
||
};
|
||
}
|
||
|
||
return {
|
||
metrics: {
|
||
ready,
|
||
...(await readPageMetrics(page, ready)),
|
||
...transferred,
|
||
},
|
||
// builds without the mark have no phases
|
||
phases:
|
||
appStart === undefined
|
||
? {}
|
||
: { appCode: Math.min(appStart, ready), editor: ready },
|
||
buildSha: validBuildSha,
|
||
};
|
||
} finally {
|
||
await context.close();
|
||
}
|
||
}
|
||
|
||
/** A free local port, from the OS. */
|
||
const getFreePort = () =>
|
||
new Promise((resolve, reject) => {
|
||
const server = net.createServer();
|
||
server.once("error", reject);
|
||
server.listen(0, () => {
|
||
const { port } = server.address();
|
||
server.close(() => resolve(port));
|
||
});
|
||
});
|
||
|
||
const git = (...gitArgs) =>
|
||
execFileSync("git", gitArgs, { cwd: REPO_ROOT, encoding: "utf8" }).trim();
|
||
|
||
/** Runs a command with its output kept back, but for the end of it when it fails. */
|
||
function runQuietly(command, commandArgs, options, failure) {
|
||
try {
|
||
execFileSync(command, commandArgs, {
|
||
...options,
|
||
stdio: "pipe",
|
||
maxBuffer: 256 * 1024 * 1024,
|
||
});
|
||
} catch (error) {
|
||
console.error(
|
||
String(error.stderr ?? error)
|
||
.trim()
|
||
.split("\n")
|
||
.slice(-20)
|
||
.join("\n"),
|
||
);
|
||
throw new Error(failure);
|
||
}
|
||
}
|
||
|
||
/** This checkout: its commit, and whether it has changes on top. */
|
||
function getThisCheckout() {
|
||
const sha = git("rev-parse", "HEAD");
|
||
return {
|
||
dir: REPO_ROOT,
|
||
sha,
|
||
label: `this checkout (${sha.slice(0, 7)}${
|
||
git("status", "--porcelain") ? " + uncommitted changes" : ""
|
||
})`,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* `revision` checked out into a temporary worktree with its own dependencies
|
||
* (they may differ from this checkout's), removed when the script ends.
|
||
*/
|
||
function checkOut(revision) {
|
||
const sha = git("rev-parse", "--verify", `${revision}^{commit}`);
|
||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "excalidraw-perf-"));
|
||
const dir = path.join(tmpDir, "excalidraw");
|
||
const label = sha.startsWith(revision)
|
||
? sha.slice(0, 7)
|
||
: `${revision} (${sha.slice(0, 7)})`;
|
||
|
||
console.info(`⏱️ Checking out ${label} and installing its dependencies…`);
|
||
process.on("exit", () => {
|
||
try {
|
||
execFileSync("git", ["worktree", "remove", "--force", dir], {
|
||
cwd: REPO_ROOT,
|
||
stdio: "ignore",
|
||
});
|
||
} catch {
|
||
// the script ended before git created it
|
||
}
|
||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||
});
|
||
git("worktree", "add", "--quiet", "--detach", dir, sha);
|
||
runQuietly(
|
||
"yarn",
|
||
["install", "--frozen-lockfile", "--prefer-offline"],
|
||
// no git hooks for the worktree
|
||
{ cwd: dir, env: { ...process.env, HUSKY: "0" } },
|
||
`Installing the dependencies of ${label} failed`,
|
||
);
|
||
|
||
return { dir, sha, label };
|
||
}
|
||
|
||
/**
|
||
* Builds a checkout's app for production — as Vercel does (`build:app`),
|
||
* minus linting — and serves it with `vite preview`: its url, and how to stop
|
||
* the server.
|
||
*/
|
||
async function serveLocalBuild({ dir, sha, label }) {
|
||
const appDir = path.join(dir, "excalidraw-app");
|
||
const vite = path.join(dir, "node_modules/vite/bin/vite.js");
|
||
|
||
console.info(`⏱️ Building ${label}…`);
|
||
runQuietly(
|
||
process.execPath,
|
||
[vite, "build"],
|
||
{
|
||
cwd: appDir,
|
||
env: {
|
||
...process.env,
|
||
VITE_APP_GIT_SHA: sha,
|
||
VITE_APP_ENABLE_TRACKING: "true",
|
||
VITE_APP_ENABLE_ESLINT: "false",
|
||
},
|
||
},
|
||
`Building ${label} failed (\`yarn build:app\` shows why)`,
|
||
);
|
||
|
||
const port = await getFreePort();
|
||
const server = spawn(
|
||
process.execPath,
|
||
[vite, "preview", "--port", String(port), "--strictPort"],
|
||
{ cwd: appDir, stdio: "ignore" },
|
||
);
|
||
// however the script ends
|
||
process.on("exit", () => server.kill());
|
||
const url = `http://localhost:${port}`;
|
||
for (let attempt = 0; ; attempt++) {
|
||
const isUp = await fetch(url)
|
||
.then((response) => response.ok)
|
||
.catch(() => false);
|
||
if (isUp) {
|
||
return { url, stop: () => server.kill() };
|
||
}
|
||
if (attempt > 60 || server.exitCode !== null) {
|
||
server.kill();
|
||
throw new Error("`vite preview` didn't start");
|
||
}
|
||
await sleep(500);
|
||
}
|
||
}
|
||
|
||
// so a stopped run still removes its worktree and servers
|
||
for (const signal of ["SIGINT", "SIGTERM"]) {
|
||
process.once(signal, () => process.exit(1));
|
||
}
|
||
|
||
/** The metrics the runner records, in the admin's order (`PERF_METRICS` in Excalidraw+), and their units. */
|
||
const METRIC_UNITS = {
|
||
ready: "ms",
|
||
ttfb: "ms",
|
||
fcp: "ms",
|
||
lcp: "ms",
|
||
tbt: "ms",
|
||
cls: "score",
|
||
jsBytes: "bytes",
|
||
transferBytes: "bytes",
|
||
requests: "count",
|
||
frameMean: "ms",
|
||
frameP95: "ms",
|
||
longFrames: "count",
|
||
};
|
||
|
||
/** Linear-interpolated quantile of ascending `sorted` (the admin's too). */
|
||
const quantile = (sorted, q) => {
|
||
const position = (sorted.length - 1) * q;
|
||
const lower = Math.floor(position);
|
||
const upper = Math.ceil(position);
|
||
return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower);
|
||
};
|
||
|
||
const summarize = (values) => {
|
||
const sorted = [...values].sort((a, b) => a - b);
|
||
return {
|
||
median: quantile(sorted, 0.5),
|
||
p25: quantile(sorted, 0.25),
|
||
p75: quantile(sorted, 0.75),
|
||
};
|
||
};
|
||
|
||
/** Below it, a difference in timings is unlikely to be the loads' noise. */
|
||
const SIGNIFICANCE = 0.05;
|
||
|
||
/**
|
||
* The two-sided p-value of the Mann–Whitney U test: how likely the two
|
||
* samples' ranks are this far apart if the builds didn't differ — no
|
||
* assumption on how timings spread, fit for a few loads per side. Exact (by
|
||
* counting the rank arrangements); ties get mid-ranks.
|
||
*/
|
||
function mannWhitneyP(a, b) {
|
||
const ranked = [
|
||
...a.map((value) => ({ value, inA: true })),
|
||
...b.map((value) => ({ value, inA: false })),
|
||
].sort((x, y) => x.value - y.value);
|
||
let rankSumA = 0;
|
||
for (let i = 0; i < ranked.length; ) {
|
||
let j = i;
|
||
while (j + 1 < ranked.length && ranked[j + 1].value === ranked[i].value) {
|
||
j++;
|
||
}
|
||
// ranks i + 1 … j + 1, shared
|
||
const rank = (i + j + 2) / 2;
|
||
for (let k = i; k <= j; k++) {
|
||
if (ranked[k].inA) {
|
||
rankSumA += rank;
|
||
}
|
||
}
|
||
i = j + 1;
|
||
}
|
||
const m = a.length;
|
||
const n = b.length;
|
||
const uA = rankSumA - (m * (m + 1)) / 2;
|
||
const u = Math.floor(Math.min(uA, m * n - uA));
|
||
|
||
// arrangements of m and n values with the statistic `value`
|
||
const memo = new Map();
|
||
const count = (value, m, n) => {
|
||
if (value < 0) {
|
||
return 0;
|
||
}
|
||
if (!m || !n) {
|
||
return value === 0 ? 1 : 0;
|
||
}
|
||
const key = `${value},${m},${n}`;
|
||
if (!memo.has(key)) {
|
||
memo.set(key, count(value - n, m - 1, n) + count(value, m, n - 1));
|
||
}
|
||
return memo.get(key);
|
||
};
|
||
let atMost = 0;
|
||
for (let value = 0; value <= u; value++) {
|
||
atMost += count(value, m, n);
|
||
}
|
||
let arrangements = 1;
|
||
for (let i = 1; i <= m; i++) {
|
||
arrangements = (arrangements * (n + i)) / i;
|
||
}
|
||
return Math.min(1, (2 * atMost) / arrangements);
|
||
}
|
||
|
||
/**
|
||
* Every metric is better lower. Timings differ when the Mann–Whitney test
|
||
* says so (`SIGNIFICANCE`); bytes and requests don't vary from load to load,
|
||
* so any difference in them counts.
|
||
*/
|
||
function getVerdict(baseValues, nextValues) {
|
||
const isConstant = (values) => values.every((value) => value === values[0]);
|
||
const direction = Math.sign(median(nextValues) - median(baseValues));
|
||
const p =
|
||
isConstant(baseValues) && isConstant(nextValues)
|
||
? direction
|
||
? 0
|
||
: 1
|
||
: mannWhitneyP(baseValues, nextValues);
|
||
return {
|
||
p,
|
||
verdict:
|
||
p < SIGNIFICANCE && direction
|
||
? direction > 0
|
||
? "worse"
|
||
: "better"
|
||
: "~",
|
||
};
|
||
}
|
||
|
||
const median = (values) =>
|
||
quantile(
|
||
[...values].sort((a, b) => a - b),
|
||
0.5,
|
||
);
|
||
|
||
/** One row per scenario and metric both targets measured. */
|
||
function compareTargets(scenarios, base, next) {
|
||
const rows = [];
|
||
for (const scenario of scenarios) {
|
||
const failed = (target) =>
|
||
target.samples.filter((s) => s.scenario === scenario.id && s.error)
|
||
.length;
|
||
for (const [metric, unit] of Object.entries(METRIC_UNITS)) {
|
||
const values = (target) =>
|
||
target.samples.flatMap((sample) =>
|
||
sample.scenario === scenario.id &&
|
||
!sample.error &&
|
||
sample.metrics[metric] !== undefined
|
||
? [sample.metrics[metric]]
|
||
: [],
|
||
);
|
||
const baseValues = values(base);
|
||
const nextValues = values(next);
|
||
if (!baseValues.length || !nextValues.length) {
|
||
continue;
|
||
}
|
||
const baseStats = summarize(baseValues);
|
||
const nextStats = summarize(nextValues);
|
||
const delta = nextStats.median - baseStats.median;
|
||
rows.push({
|
||
scenario: scenario.id,
|
||
metric,
|
||
unit,
|
||
base: baseStats,
|
||
next: nextStats,
|
||
delta,
|
||
deltaPercent: baseStats.median
|
||
? (delta / baseStats.median) * 100
|
||
: null,
|
||
...getVerdict(baseValues, nextValues),
|
||
baseFailed: failed(base),
|
||
nextFailed: failed(next),
|
||
});
|
||
}
|
||
}
|
||
return rows;
|
||
}
|
||
|
||
const formatValue = (value, unit) =>
|
||
unit === "ms"
|
||
? `${
|
||
value < 100 ? value.toFixed(1) : Math.round(value).toLocaleString("en")
|
||
} ms`
|
||
: unit === "bytes"
|
||
? `${(value / 1024).toLocaleString("en", {
|
||
minimumFractionDigits: 1,
|
||
maximumFractionDigits: 1,
|
||
})} KiB`
|
||
: unit === "score"
|
||
? value.toFixed(3)
|
||
: String(Math.round(value * 10) / 10);
|
||
|
||
const formatSigned = (value, format) =>
|
||
`${value > 0 ? "+" : value < 0 ? "−" : "±"}${format(Math.abs(value))}`;
|
||
|
||
const VERDICT_COLORS = { worse: 31, better: 32 };
|
||
|
||
/** The comparison as an aligned table, the verdicts colored on a terminal. */
|
||
function printComparison(rows) {
|
||
const color = (text, verdict) =>
|
||
process.stdout.isTTY && VERDICT_COLORS[verdict]
|
||
? `\x1b[${VERDICT_COLORS[verdict]}m${text}\x1b[0m`
|
||
: text;
|
||
const header = [
|
||
"Scenario",
|
||
"Metric",
|
||
"Base",
|
||
"New",
|
||
"Change",
|
||
"Change %",
|
||
"Base p25–p75",
|
||
"New p25–p75",
|
||
"p",
|
||
"",
|
||
];
|
||
const lines = rows.map((row, index) => [
|
||
index && rows[index - 1].scenario === row.scenario ? "" : row.scenario,
|
||
row.metric,
|
||
formatValue(row.base.median, row.unit),
|
||
formatValue(row.next.median, row.unit),
|
||
formatSigned(row.delta, (value) => formatValue(value, row.unit)),
|
||
row.deltaPercent === null
|
||
? "—"
|
||
: formatSigned(row.deltaPercent, (value) => `${value.toFixed(1)} %`),
|
||
`${formatValue(row.base.p25, row.unit)} – ${formatValue(
|
||
row.base.p75,
|
||
row.unit,
|
||
)}`,
|
||
`${formatValue(row.next.p25, row.unit)} – ${formatValue(
|
||
row.next.p75,
|
||
row.unit,
|
||
)}`,
|
||
row.p < 0.001 ? "<0.001" : row.p.toFixed(3),
|
||
row.verdict,
|
||
]);
|
||
const widths = header.map((title, column) =>
|
||
Math.max(title.length, ...lines.map((line) => line[column].length)),
|
||
);
|
||
// the names left-aligned, the numbers right-aligned
|
||
const pad = (cell, column) =>
|
||
column < 2 || column === header.length - 1
|
||
? cell.padEnd(widths[column])
|
||
: cell.padStart(widths[column]);
|
||
|
||
console.info(header.map(pad).join(" "));
|
||
console.info(widths.map((width) => "─".repeat(width)).join(" "));
|
||
rows.forEach((row, index) =>
|
||
console.info(color(lines[index].map(pad).join(" "), row.verdict)),
|
||
);
|
||
}
|
||
|
||
const CSV_COLUMNS = [
|
||
"scenario",
|
||
"metric",
|
||
"unit",
|
||
"base_median",
|
||
"base_p25",
|
||
"base_p75",
|
||
"new_median",
|
||
"new_p25",
|
||
"new_p75",
|
||
"change",
|
||
"change_percent",
|
||
"p_value",
|
||
"verdict",
|
||
"base_failed",
|
||
"new_failed",
|
||
];
|
||
|
||
const toCsv = (rows) =>
|
||
[
|
||
CSV_COLUMNS.join(","),
|
||
...rows.map((row) =>
|
||
[
|
||
row.scenario,
|
||
row.metric,
|
||
row.unit,
|
||
row.base.median,
|
||
row.base.p25,
|
||
row.base.p75,
|
||
row.next.median,
|
||
row.next.p25,
|
||
row.next.p75,
|
||
row.delta,
|
||
row.deltaPercent ?? "",
|
||
row.p,
|
||
row.verdict,
|
||
row.baseFailed,
|
||
row.nextFailed,
|
||
].join(","),
|
||
),
|
||
].join("\n");
|
||
|
||
const scenarios = args.scenario
|
||
? SCENARIOS.filter(({ id }) => args.scenario.split(",").includes(id))
|
||
: SCENARIOS;
|
||
|
||
/** A build to measure: its url, its commit when it's built here, and its loads. */
|
||
const target = (name, { url, label = url, commitSha = null }) => ({
|
||
name,
|
||
url,
|
||
label,
|
||
commitSha,
|
||
samples: [],
|
||
buildSha: null,
|
||
});
|
||
|
||
const stopServers = [];
|
||
|
||
/** A checkout's build, served here. */
|
||
async function localTarget(name, checkout) {
|
||
const { url, stop } = await serveLocalBuild(checkout);
|
||
stopServers.push(stop);
|
||
return target(name, {
|
||
url,
|
||
label: `${checkout.label} at ${url}`,
|
||
commitSha: checkout.sha,
|
||
});
|
||
}
|
||
|
||
const isUrl = (value) => /^https?:\/\//.test(value);
|
||
|
||
// with --compare, the base and the new build alternate: whichever loads first
|
||
// switches every iteration, so the machine drifting (warming up, other work)
|
||
// weighs on both alike
|
||
const targets = args.compare
|
||
? [
|
||
isUrl(args.base)
|
||
? target("base", { url: withoutSlash(args.base) })
|
||
: await localTarget("base", checkOut(args.base)),
|
||
args.url
|
||
? target("new", { url: APP_URL })
|
||
: await localTarget("new", getThisCheckout()),
|
||
]
|
||
: [target(null, { url: APP_URL })];
|
||
|
||
console.info(
|
||
args.compare
|
||
? `⏱️ Comparing ${targets[1].label} with ${targets[0].label}, ${ITERATIONS} loads each per scenario, alternating…`
|
||
: `⏱️ Measuring ${APP_URL}, ${ITERATIONS} loads per scenario…`,
|
||
);
|
||
|
||
const browser = await chromium.launch();
|
||
const browserVersion = browser.version();
|
||
|
||
try {
|
||
// the browser's first load is slower than the next ones: one of each build, not recorded
|
||
for (const target of targets) {
|
||
await measureOnce(browser, scenarios[0], target.url).catch(() => {});
|
||
}
|
||
for (const scenario of scenarios) {
|
||
for (let iteration = 1; iteration <= ITERATIONS; iteration++) {
|
||
const order = iteration % 2 ? targets : [...targets].reverse();
|
||
for (const target of order) {
|
||
const label = `${scenario.id}${
|
||
target.name ? ` ${target.name}` : ""
|
||
} #${iteration}`;
|
||
try {
|
||
await sleep(BETWEEN_LOADS_MS);
|
||
const load = await measureOnce(browser, scenario, target.url);
|
||
target.buildSha ??= load.buildSha;
|
||
target.samples.push({
|
||
scenario: scenario.id,
|
||
iteration,
|
||
metrics: load.metrics,
|
||
phases: load.phases,
|
||
error: null,
|
||
});
|
||
console.info(
|
||
`⏱️ ${label}: ${
|
||
scenario.interact
|
||
? `mean frame ${Math.round(
|
||
load.metrics.frameMean,
|
||
)} ms, p95 ${Math.round(load.metrics.frameP95)} ms, ${
|
||
load.metrics.longFrames
|
||
} long`
|
||
: `ready in ${Math.round(load.metrics.ready)} ms`
|
||
}`,
|
||
);
|
||
} catch (error) {
|
||
// a failed load is a sample with its error, so one flaky load doesn't lose the run
|
||
const message = (
|
||
error instanceof Error ? error.message : String(error)
|
||
).split("\n")[0];
|
||
target.samples.push({
|
||
scenario: scenario.id,
|
||
iteration,
|
||
metrics: {},
|
||
phases: {},
|
||
error: message,
|
||
});
|
||
console.warn(`⏱️ ${label} failed: ${message}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} finally {
|
||
await browser.close();
|
||
stopServers.forEach((stop) => stop());
|
||
}
|
||
|
||
if (targets.some(({ samples }) => !samples.some(({ error }) => !error))) {
|
||
console.error("⏱️ Every page load failed");
|
||
process.exit(1);
|
||
}
|
||
|
||
const [base, next] = args.compare ? targets : [null, targets[0]];
|
||
// with --compare, the base build's loads too (`post.mjs` stores them with the run)
|
||
const run = {
|
||
appUrl: next.url,
|
||
buildSha: next.buildSha,
|
||
environment: {
|
||
...PERF_ENVIRONMENT,
|
||
runner: process.env.RUNNER_NAME ?? null,
|
||
cpu: os.cpus()[0]?.model ?? null,
|
||
browser: `chromium ${browserVersion}`,
|
||
iterations: ITERATIONS,
|
||
},
|
||
samples: next.samples,
|
||
...(base && {
|
||
base: {
|
||
// a url's: the build it served
|
||
commitSha: base.commitSha ?? base.buildSha,
|
||
buildSha: base.buildSha,
|
||
samples: base.samples,
|
||
},
|
||
}),
|
||
};
|
||
|
||
if (args.compare) {
|
||
const rows = compareTargets(scenarios, base, next);
|
||
const describe = ({ label, buildSha }) =>
|
||
`${label}${buildSha ? ` (build ${buildSha.slice(0, 7)})` : ""}`;
|
||
|
||
console.info(`\nBase: ${describe(base)}\nNew: ${describe(next)}\n`);
|
||
printComparison(rows);
|
||
for (const row of rows.filter(
|
||
(row, index) =>
|
||
(row.baseFailed || row.nextFailed) &&
|
||
rows.findIndex(({ scenario }) => scenario === row.scenario) === index,
|
||
)) {
|
||
console.warn(
|
||
`⏱️ ${row.scenario}: ${row.baseFailed} base and ${row.nextFailed} new loads failed`,
|
||
);
|
||
}
|
||
if (args.out) {
|
||
fs.writeFileSync(
|
||
args.out,
|
||
args.out.endsWith(".csv")
|
||
? `${toCsv(rows)}\n`
|
||
: JSON.stringify(run, null, 2),
|
||
);
|
||
console.info(`\nWrote ${args.out}`);
|
||
}
|
||
} else if (args.out) {
|
||
fs.writeFileSync(args.out, JSON.stringify(run, null, 2));
|
||
} else {
|
||
console.info(JSON.stringify(run, null, 2));
|
||
}
|