feat(tests): screenshot regression + Discord review, perf-tracked
New tooling in tests/tools/screenshots/ (TypeScript via tsx): - compare.ts: one pixelmatch engine (AA-excluded), connected-component "where to look" boxes, old|new+boxes|heatmap triptych, per-engine floors. - promote.ts: churn-free updater — overwrite a baseline only when decoded pixels differ beyond the floor, copying CI bytes verbatim (no re-encode churn); pulls a CI run via `gh run download` or a local --from dir. - post-discord.ts: always-on CI-on-main report (SHA + e2e status + the track-only runtime-perf table), then screenshot triptychs, batched + size-capped + flood-collapsed + 429-aware. - perf-report.ts: perf table with Δ vs the previous main run (via gh). - changelog.ts: no-build git-history baseline differ (Discord trigger B). - noise.ts / gen-manifest.ts: calibration + manifest generation. CI wiring: - wasm-build.yml: post-test step runs the gate + report on the already- produced test-results (no extra build); report-only (continue-on-error), posts only on push to main, inert without DISCORD_WEBHOOK_URL. - ci-ubicloud.yml: secrets: inherit (pass the webhook through). - screenshot-changelog.yml: ~30s no-build changelog on baseline changes. screenshot-manifest.json: canonical 354-name set + best-effort engine tags (313 chromium-swiftshader / 41 firefox-llvmpipe). Normalize scale:'device'->'css' across 18 spec files (no-op at CI DSF=1) so committed baselines are uniformly css-scaled. Design: CI's Linux render is the single source of truth; no pinned container (accept rare env drift -> re-promote); dev commits via promote. Replaces the byte-cmp compare-screenshots.sh + file-size-proxy update-baseline-screenshots.sh (kept for now until the first re-baseline). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Vaczi
parent
05127af431
commit
13551f4f22
36 changed files with 3186 additions and 63 deletions
164
tests/tools/screenshots/perf-report.ts
Normal file
164
tests/tools/screenshots/perf-report.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
/**
|
||||
* Renders the track-only runtime-perf block for the CI-on-main Discord comment.
|
||||
*
|
||||
* The perf e2e (tests/kicad/{eeschema,pcbnew}-perf.spec.ts) already writes
|
||||
* test-results/perf-{app}.json — schema { app, when, loadMs, openMs, fps:[{throttle,fps}] }.
|
||||
* We read those, fetch the PREVIOUS successful main run's perf via `gh run
|
||||
* download` (so we can show a Δ without committing a baseline — stays
|
||||
* no-write-back), and format an aligned monospace table (Discord doesn't render
|
||||
* markdown tables, so it goes in a ``` code block).
|
||||
*
|
||||
* Track-only: nothing here gates the build. A regression past REGRESSION_PCT on
|
||||
* the stable metrics (loadMs/openMs) is only flagged (a `*`), never failed. FPS
|
||||
* is CPU-bound/noisy on CI's headless SwiftShader path, so it's shown but marked
|
||||
* indicative.
|
||||
*
|
||||
* CLI (from tests/):
|
||||
* tsx tools/screenshots/perf-report.ts [--results DIR] [--prev DIR] [--repo owner/repo]
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { RESULTS_DIR } from './config';
|
||||
|
||||
export const PERF_APPS = ['eeschema', 'pcbnew'] as const;
|
||||
const REGRESSION_PCT = 10; // stable-metric regression past this is flagged with `*`
|
||||
const CI_WORKFLOW = 'ci-ubicloud.yml';
|
||||
|
||||
export type Fps = { throttle: number; fps: number };
|
||||
export type PerfData = { app: string; when?: string; loadMs: number; openMs: number; fps: Fps[] };
|
||||
|
||||
export function readPerf(dir: string): Map<string, PerfData> {
|
||||
const out = new Map<string, PerfData>();
|
||||
for (const app of PERF_APPS) {
|
||||
const p = path.join(dir, `perf-${app}.json`);
|
||||
if (!fs.existsSync(p)) continue;
|
||||
try {
|
||||
out.set(app, JSON.parse(fs.readFileSync(p, 'utf8')) as PerfData);
|
||||
} catch (e) {
|
||||
console.warn(`[perf] could not parse ${p}: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort fetch of the previous successful main CI run's perf JSONs into a
|
||||
* temp dir. Returns the dir, or null if gh is unavailable / no prior run.
|
||||
* `currentSha` is skipped so a re-run doesn't diff against itself.
|
||||
*/
|
||||
export function fetchPreviousPerf(repo: string | undefined, currentSha: string | undefined): string | null {
|
||||
try {
|
||||
const gh = (args: string[]) =>
|
||||
execFileSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
const repoArgs = repo ? ['--repo', repo] : [];
|
||||
const runs = JSON.parse(
|
||||
gh([
|
||||
'run', 'list', '--workflow', CI_WORKFLOW, '--branch', 'main', '--status', 'success',
|
||||
'--limit', '15', '--json', 'databaseId,headSha', ...repoArgs,
|
||||
])
|
||||
) as Array<{ databaseId: number; headSha: string }>;
|
||||
const prev = runs.find((r) => r.headSha !== currentSha);
|
||||
if (!prev) return null;
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'perf-prev-'));
|
||||
// Artifact is named ubicloud-e2e-<run_id>; -D extracts it under tmp/<artifact>/...
|
||||
gh(['run', 'download', String(prev.databaseId), '-D', tmp, ...repoArgs]);
|
||||
// Find the dir that actually holds the perf-*.json (artifact nests test-results/).
|
||||
const hit = findPerfDir(tmp);
|
||||
return hit;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function findPerfDir(root: string): string | null {
|
||||
const stack = [root];
|
||||
while (stack.length) {
|
||||
const dir = stack.pop()!;
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (entries.some((e) => e.isFile() && /^perf-\w+\.json$/.test(e.name))) return dir;
|
||||
for (const e of entries) if (e.isDirectory()) stack.push(path.join(dir, e.name));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function pct(cur: number, prev: number): number {
|
||||
return prev === 0 ? 0 : ((cur - prev) / prev) * 100;
|
||||
}
|
||||
|
||||
/** Format a stable-metric value with a Δ vs previous (lower is better). */
|
||||
function fmtMetric(cur: number, prev?: number): string {
|
||||
if (prev === undefined) return `${cur}`;
|
||||
const p = pct(cur, prev);
|
||||
const arrow = p < 0 ? '▼' : p > 0 ? '▲' : '·';
|
||||
const flag = p > REGRESSION_PCT ? '*' : ''; // regression (slower) beyond threshold
|
||||
return `${cur} ${arrow}${Math.abs(p).toFixed(0)}%${flag}`;
|
||||
}
|
||||
|
||||
function fmtFps(fps: Fps[]): string {
|
||||
return [1, 4, 6].map((t) => {
|
||||
const hit = fps.find((f) => f.throttle === t);
|
||||
return hit ? Math.round(hit.fps) : '–';
|
||||
}).join('/');
|
||||
}
|
||||
|
||||
function pad(s: string, n: number): string {
|
||||
return s.length >= n ? s : s + ' '.repeat(n - s.length);
|
||||
}
|
||||
|
||||
export type PerfReport = { block: string; regressed: boolean };
|
||||
|
||||
/** Build the fenced code-block perf table for the Discord comment. */
|
||||
export function buildPerfReport(opts: { resultsDir?: string; prevDir?: string | null } = {}): PerfReport {
|
||||
const cur = readPerf(opts.resultsDir ?? RESULTS_DIR);
|
||||
if (cur.size === 0) return { block: '', regressed: false };
|
||||
const prev = opts.prevDir ? readPerf(opts.prevDir) : new Map<string, PerfData>();
|
||||
|
||||
const headers = ['app', 'loadMs (Δ)', 'openMs (Δ)', 'FPS 1/4/6'];
|
||||
const rows: string[][] = [];
|
||||
let regressed = false;
|
||||
for (const app of PERF_APPS) {
|
||||
const c = cur.get(app);
|
||||
if (!c) continue;
|
||||
const p = prev.get(app);
|
||||
const loadCell = fmtMetric(c.loadMs, p?.loadMs);
|
||||
const openCell = fmtMetric(c.openMs, p?.openMs);
|
||||
if (loadCell.endsWith('*') || openCell.endsWith('*')) regressed = true;
|
||||
rows.push([app, loadCell, openCell, fmtFps(c.fps)]);
|
||||
}
|
||||
if (rows.length === 0) return { block: '', regressed: false };
|
||||
|
||||
const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
|
||||
const line = (cells: string[]) => cells.map((c, i) => pad(c, widths[i])).join(' ');
|
||||
const body = [line(headers), rows.map((r) => line(r)).join('\n')].join('\n');
|
||||
const footnote = `${prev.size ? 'Δ vs previous main run. ' : 'no prior main run for Δ. '}` +
|
||||
`* = >${REGRESSION_PCT}% slower (track-only, non-gating). FPS is CI-headless — indicative only.`;
|
||||
return { block: '**Runtime perf** (eeschema + pcbnew)\n```\n' + body + '\n```\n' + footnote, regressed };
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--results') out.results = argv[++i];
|
||||
else if (a === '--prev') out.prev = argv[++i];
|
||||
else if (a === '--repo') out.repo = argv[++i];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const prevDir = args.prev ?? fetchPreviousPerf(args.repo || process.env.GITHUB_REPOSITORY, process.env.GITHUB_SHA);
|
||||
const { block, regressed } = buildPerfReport({ resultsDir: args.results, prevDir });
|
||||
process.stdout.write((block || '(no perf-*.json found)') + '\n');
|
||||
if (regressed) console.error('[perf] a stable metric regressed >10% vs previous main (track-only, not failing)');
|
||||
}
|
||||
|
||||
if (require.main === module) main();
|
||||
Loading…
Reference in a new issue