feat(tests): caption posted screenshots (name + spec); raise drift floor to 0.5%
Bake a caption strip onto every posted screenshot composite — status + name + the spec that produces it — for changed, added, and removed; removed now shows the old baseline image (was a text-only line). Zero native-dep: an embedded public-domain 8x8 bitmap font (font8x8.ts) rendered by image-ops `withBottomLabel`; the name→spec attribution is factored out of gen-manifest into a shared `spec-map.ts` resolver. Bottom strip, colour per status (green/red/orange). Applies in both compare (drift gate) and changelog (git-history diff), and post-discord now attaches the captioned removed images. Also raise the per-engine drift floor 0.2% → 0.5% (changedRatio) to absorb the sub-1% inter-run flakiness seen after the re-baseline, while still catching real localized changes. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Vaczi
parent
4330c9344e
commit
9787efc2c9
9 changed files with 360 additions and 101 deletions
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"_note": "engine tags are best-effort (gen-manifest.ts); the name list is authoritative. Refine engines after calibration.",
|
||||
"_note": "engine tags are best-effort (spec-map.ts); the name list is authoritative. Refine engines after calibration.",
|
||||
"screenshots": [
|
||||
{
|
||||
"name": "01-loading.png",
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@ import * as fs from 'fs';
|
|||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { BASELINE_DIRS, DIFF_OUT_DIR, floorFor } from './config';
|
||||
import { PNG } from 'pngjs';
|
||||
import { BASELINE_DIRS, DIFF_OUT_DIR, floorFor, labelText, LABEL } from './config';
|
||||
import { comparePair, type Report } from './compare';
|
||||
import { loadPng, savePng } from './image-ops';
|
||||
import { loadPng, savePng, withBottomLabel } from './image-ops';
|
||||
import { buildSpecResolver } from './spec-map';
|
||||
import { buildAttachments, paginate, postMessage } from './post-discord';
|
||||
|
||||
function git(root: string, args: string[]): string {
|
||||
|
|
@ -77,6 +79,23 @@ async function main(): Promise<void> {
|
|||
unchangedCount: 0,
|
||||
driftLikely: false,
|
||||
};
|
||||
const { specFor } = buildSpecResolver(cwd);
|
||||
|
||||
// Load a git blob (a committed PNG at `rev`) as a decoded PNG, or null if absent.
|
||||
const blobPng = (rev: string, p: string): PNG | null => {
|
||||
const b = gitBlob(repoRoot, rev, p);
|
||||
if (!b) return null;
|
||||
const f = path.join(tmp, `${rev.slice(0, 7)}-${path.basename(p)}`);
|
||||
fs.writeFileSync(f, b);
|
||||
return loadPng(f);
|
||||
};
|
||||
// Save a single captioned image (added/removed) and record it in the report.
|
||||
const saveSingle = (img: PNG, name: string, status: 'added' | 'removed'): void => {
|
||||
const suffix = status === 'added' ? 'added' : 'removed';
|
||||
const rel = path.join(DIFF_OUT_DIR, `${name}.${suffix}.png`);
|
||||
savePng(path.join(cwd, rel), withBottomLabel(img, labelText(status, name, specFor(name)), LABEL.colors[status]));
|
||||
report[status].push({ name, image: rel });
|
||||
};
|
||||
|
||||
for (const line of diff.split('\n')) {
|
||||
const parts = line.split('\t');
|
||||
|
|
@ -86,28 +105,25 @@ async function main(): Promise<void> {
|
|||
const name = path.basename(repoPath);
|
||||
|
||||
if (status === 'A' || status === 'R') {
|
||||
const buf = gitBlob(repoRoot, head, repoPath);
|
||||
if (buf) {
|
||||
const rel = path.join(DIFF_OUT_DIR, `${name}.added.png`);
|
||||
fs.writeFileSync(path.join(cwd, rel), buf);
|
||||
report.added.push({ name, image: rel });
|
||||
const img = blobPng(head, repoPath);
|
||||
if (img) saveSingle(img, name, 'added');
|
||||
if (status === 'R') {
|
||||
const oldName = path.basename(oldPath);
|
||||
const oldImg = blobPng(base, oldPath);
|
||||
if (oldImg) saveSingle(oldImg, oldName, 'removed');
|
||||
}
|
||||
if (status === 'R') report.removed.push({ name: path.basename(oldPath) });
|
||||
} else if (status === 'D') {
|
||||
report.removed.push({ name });
|
||||
const img = blobPng(base, repoPath);
|
||||
if (img) saveSingle(img, name, 'removed');
|
||||
} else {
|
||||
// Modified: triptych old vs new.
|
||||
const oldBuf = gitBlob(repoRoot, base, repoPath);
|
||||
const newBuf = gitBlob(repoRoot, head, repoPath);
|
||||
if (!oldBuf || !newBuf) continue;
|
||||
const oldFile = path.join(tmp, `old-${name}`);
|
||||
const newFile = path.join(tmp, `new-${name}`);
|
||||
fs.writeFileSync(oldFile, oldBuf);
|
||||
fs.writeFileSync(newFile, newBuf);
|
||||
const { result, heatmap, triptych } = comparePair(loadPng(oldFile), loadPng(newFile), name, floorFor(name));
|
||||
// Modified: captioned triptych old vs new.
|
||||
const oldImg = blobPng(base, repoPath);
|
||||
const newImg = blobPng(head, repoPath);
|
||||
if (!oldImg || !newImg) continue;
|
||||
const { result, heatmap, triptych } = comparePair(oldImg, newImg, name, floorFor(name));
|
||||
const triptychRel = path.join(DIFF_OUT_DIR, `${name}.triptych.png`);
|
||||
const heatmapRel = path.join(DIFF_OUT_DIR, `${name}.heatmap.png`);
|
||||
savePng(path.join(cwd, triptychRel), triptych);
|
||||
savePng(path.join(cwd, triptychRel), withBottomLabel(triptych, labelText('changed', name, specFor(name)), LABEL.colors.changed));
|
||||
savePng(path.join(cwd, heatmapRel), heatmap);
|
||||
report.changed.push({ ...result, triptych: triptychRel, heatmap: heatmapRel });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,8 +24,11 @@ import {
|
|||
type Manifest,
|
||||
floorFor,
|
||||
isIgnored,
|
||||
labelText,
|
||||
LABEL,
|
||||
} from './config';
|
||||
import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, type Box } from './image-ops';
|
||||
import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, withBottomLabel, type Box } from './image-ops';
|
||||
import { buildSpecResolver } from './spec-map';
|
||||
|
||||
export type PairVerdict = 'unchanged' | 'changed';
|
||||
|
||||
|
|
@ -79,7 +82,7 @@ export type Report = {
|
|||
generatedFor: string | null;
|
||||
changed: ChangedEntry[];
|
||||
added: Array<{ name: string; image: string }>;
|
||||
removed: Array<{ name: string }>;
|
||||
removed: Array<{ name: string; image: string }>;
|
||||
unchangedCount: number;
|
||||
/** many changes, mostly drift-like ⇒ probably a host Mesa/font refresh; re-promote rather than debug */
|
||||
driftLikely: boolean;
|
||||
|
|
@ -130,6 +133,7 @@ export function classify(root: string, sha: string | null): Report {
|
|||
const manifest = loadManifest(root);
|
||||
const outDir = path.join(root, DIFF_OUT_DIR);
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const { specFor } = buildSpecResolver(root); // name → spec, for the caption strip
|
||||
|
||||
const report: Report = {
|
||||
generatedFor: sha,
|
||||
|
|
@ -148,7 +152,11 @@ export function classify(root: string, sha: string | null): Report {
|
|||
// an intentional removal. (The stronger "did the spec actually run" cross-check
|
||||
// against the Playwright JSON report lands with the manifest work.)
|
||||
if (manifest?.screenshots.some((e) => e.name === name)) {
|
||||
report.removed.push({ name });
|
||||
// Removed now gets a captioned image (the old baseline) so it's visible in Discord.
|
||||
const imageRel = path.join(DIFF_OUT_DIR, `${name}.removed.png`);
|
||||
const labeled = withBottomLabel(loadPng(baselinePath), labelText('removed', name, specFor(name)), LABEL.colors.removed);
|
||||
savePng(path.join(root, imageRel), labeled);
|
||||
report.removed.push({ name, image: imageRel });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
|
@ -164,7 +172,7 @@ export function classify(root: string, sha: string | null): Report {
|
|||
}
|
||||
const triptychRel = path.join(DIFF_OUT_DIR, `${name}.triptych.png`);
|
||||
const heatmapRel = path.join(DIFF_OUT_DIR, `${name}.heatmap.png`);
|
||||
savePng(path.join(root, triptychRel), triptych);
|
||||
savePng(path.join(root, triptychRel), withBottomLabel(triptych, labelText('changed', name, specFor(name)), LABEL.colors.changed));
|
||||
savePng(path.join(root, heatmapRel), heatmap);
|
||||
report.changed.push({ ...result, triptych: triptychRel, heatmap: heatmapRel });
|
||||
}
|
||||
|
|
@ -173,7 +181,8 @@ export function classify(root: string, sha: string | null): Report {
|
|||
for (const name of actuals) {
|
||||
if (baselines.has(name)) continue;
|
||||
const imageRel = path.join(DIFF_OUT_DIR, `${name}.added.png`);
|
||||
fs.copyFileSync(path.join(resultsDir, name), path.join(root, imageRel));
|
||||
const labeled = withBottomLabel(loadPng(path.join(resultsDir, name)), labelText('added', name, specFor(name)), LABEL.colors.added);
|
||||
savePng(path.join(root, imageRel), labeled);
|
||||
report.added.push({ name, image: imageRel });
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -60,16 +60,17 @@ export const TRIPTYCH = {
|
|||
* for the drift-vs-regression heuristic (broad + low-intensity ⇒ environment
|
||||
* drift, not a localized regression), not for the primary verdict.
|
||||
*
|
||||
* NOTE: these are PLACEHOLDERS. `npm run screenshots:noise` renders the suite
|
||||
* twice on the CI host and prints the real intra-CI floor per engine; set
|
||||
* `changedRatio ≈ measured × 3` from that and commit the numbers here.
|
||||
* Set to 0.5% — the re-baseline run showed 355/356 images intra-CI-stable well
|
||||
* under this, but a couple of runs since had sub-1% inter-run flakiness, so 0.5%
|
||||
* gives headroom while still catching real localized changes. (`npm run
|
||||
* screenshots:noise` on two CI renders can refine per-engine numbers later.)
|
||||
*/
|
||||
export type EngineFloor = { changedRatio: number; meanChannelGuard: number };
|
||||
|
||||
export const FLOORS: Record<string, EngineFloor> = {
|
||||
'firefox-llvmpipe': { changedRatio: 0.002, meanChannelGuard: 2.0 },
|
||||
'chromium-swiftshader': { changedRatio: 0.002, meanChannelGuard: 2.0 },
|
||||
default: { changedRatio: 0.002, meanChannelGuard: 2.0 },
|
||||
'firefox-llvmpipe': { changedRatio: 0.005, meanChannelGuard: 2.0 },
|
||||
'chromium-swiftshader': { changedRatio: 0.005, meanChannelGuard: 2.0 },
|
||||
default: { changedRatio: 0.005, meanChannelGuard: 2.0 },
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -92,6 +93,27 @@ export function isIgnored(name: string): boolean {
|
|||
return IGNORE_SCREENSHOTS.has(name);
|
||||
}
|
||||
|
||||
/** Bottom caption strip baked onto each posted composite (status + name + spec). */
|
||||
export const LABEL = {
|
||||
maxScale: 3, // bitmap-font scale; auto-fit picks the largest that fits the width
|
||||
vpad: 5,
|
||||
hpad: 8,
|
||||
text: [255, 255, 255] as [number, number, number], // white
|
||||
colors: {
|
||||
added: [46, 125, 50] as [number, number, number], // green
|
||||
removed: [198, 40, 40] as [number, number, number], // red
|
||||
changed: [239, 108, 0] as [number, number, number], // orange
|
||||
},
|
||||
};
|
||||
|
||||
export type LabelStatus = 'added' | 'removed' | 'changed';
|
||||
|
||||
/** Caption text: `CHANGED name.png · kicad/pcbnew.spec.ts` (spec omitted if unknown). */
|
||||
export function labelText(status: LabelStatus, name: string, spec: string | null): string {
|
||||
const s = status.toUpperCase();
|
||||
return spec ? `${s} ${name} · ${spec}` : `${s} ${name}`;
|
||||
}
|
||||
|
||||
export type ManifestEntry = { name: string; engine: string };
|
||||
export type Manifest = { screenshots: ManifestEntry[] };
|
||||
|
||||
|
|
|
|||
123
tests/tools/screenshots/font8x8.ts
Normal file
123
tests/tools/screenshots/font8x8.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
/**
|
||||
* Minimal 8x8 bitmap font for baking captions onto screenshot composites.
|
||||
*
|
||||
* Glyphs are the public-domain `font8x8_basic` set (Daniel Hepper's font8x8,
|
||||
* https://github.com/dhepper/font8x8 — public domain / "do whatever you want"),
|
||||
* printable ASCII 0x20–0x7E, plus a custom middle-dot (·, 0xB7) used as the label
|
||||
* separator. Public domain, so fine to embed under our GPLv3.
|
||||
*
|
||||
* Each glyph is 8 row-bytes (top→bottom). Within a row byte, bit `c` (LSB = 0) is
|
||||
* column `c`, left→right: pixel (c,row) is set iff `(byte >> c) & 1`.
|
||||
*/
|
||||
|
||||
// prettier-ignore
|
||||
const GLYPHS: Record<string, number[]> = {
|
||||
' ': [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
'!': [0x18, 0x3c, 0x3c, 0x18, 0x18, 0x00, 0x18, 0x00],
|
||||
'"': [0x36, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
'#': [0x36, 0x36, 0x7f, 0x36, 0x7f, 0x36, 0x36, 0x00],
|
||||
'$': [0x0c, 0x3e, 0x03, 0x1e, 0x30, 0x1f, 0x0c, 0x00],
|
||||
'%': [0x00, 0x63, 0x33, 0x18, 0x0c, 0x66, 0x63, 0x00],
|
||||
'&': [0x1c, 0x36, 0x1c, 0x6e, 0x3b, 0x33, 0x6e, 0x00],
|
||||
"'": [0x06, 0x06, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
'(': [0x18, 0x0c, 0x06, 0x06, 0x06, 0x0c, 0x18, 0x00],
|
||||
')': [0x06, 0x0c, 0x18, 0x18, 0x18, 0x0c, 0x06, 0x00],
|
||||
'*': [0x00, 0x66, 0x3c, 0xff, 0x3c, 0x66, 0x00, 0x00],
|
||||
'+': [0x00, 0x0c, 0x0c, 0x3f, 0x0c, 0x0c, 0x00, 0x00],
|
||||
',': [0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x0c, 0x06],
|
||||
'-': [0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00],
|
||||
'.': [0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x0c, 0x00],
|
||||
'/': [0x60, 0x30, 0x18, 0x0c, 0x06, 0x03, 0x01, 0x00],
|
||||
'0': [0x3e, 0x63, 0x73, 0x7b, 0x6f, 0x67, 0x3e, 0x00],
|
||||
'1': [0x0c, 0x0e, 0x0c, 0x0c, 0x0c, 0x0c, 0x3f, 0x00],
|
||||
'2': [0x1e, 0x33, 0x30, 0x1c, 0x06, 0x33, 0x3f, 0x00],
|
||||
'3': [0x1e, 0x33, 0x30, 0x1c, 0x30, 0x33, 0x1e, 0x00],
|
||||
'4': [0x38, 0x3c, 0x36, 0x33, 0x7f, 0x30, 0x78, 0x00],
|
||||
'5': [0x3f, 0x03, 0x1f, 0x30, 0x30, 0x33, 0x1e, 0x00],
|
||||
'6': [0x1c, 0x06, 0x03, 0x1f, 0x33, 0x33, 0x1e, 0x00],
|
||||
'7': [0x3f, 0x33, 0x30, 0x18, 0x0c, 0x0c, 0x0c, 0x00],
|
||||
'8': [0x1e, 0x33, 0x33, 0x1e, 0x33, 0x33, 0x1e, 0x00],
|
||||
'9': [0x1e, 0x33, 0x33, 0x3e, 0x30, 0x18, 0x0e, 0x00],
|
||||
':': [0x00, 0x0c, 0x0c, 0x00, 0x00, 0x0c, 0x0c, 0x00],
|
||||
';': [0x00, 0x0c, 0x0c, 0x00, 0x00, 0x0c, 0x0c, 0x06],
|
||||
'<': [0x18, 0x0c, 0x06, 0x03, 0x06, 0x0c, 0x18, 0x00],
|
||||
'=': [0x00, 0x00, 0x3f, 0x00, 0x00, 0x3f, 0x00, 0x00],
|
||||
'>': [0x06, 0x0c, 0x18, 0x30, 0x18, 0x0c, 0x06, 0x00],
|
||||
'?': [0x1e, 0x33, 0x30, 0x18, 0x0c, 0x00, 0x0c, 0x00],
|
||||
'@': [0x3e, 0x63, 0x7b, 0x7b, 0x7b, 0x03, 0x1e, 0x00],
|
||||
'A': [0x0c, 0x1e, 0x33, 0x33, 0x3f, 0x33, 0x33, 0x00],
|
||||
'B': [0x3f, 0x66, 0x66, 0x3e, 0x66, 0x66, 0x3f, 0x00],
|
||||
'C': [0x3c, 0x66, 0x03, 0x03, 0x03, 0x66, 0x3c, 0x00],
|
||||
'D': [0x1f, 0x36, 0x66, 0x66, 0x66, 0x36, 0x1f, 0x00],
|
||||
'E': [0x7f, 0x46, 0x16, 0x1e, 0x16, 0x46, 0x7f, 0x00],
|
||||
'F': [0x7f, 0x46, 0x16, 0x1e, 0x16, 0x06, 0x0f, 0x00],
|
||||
'G': [0x3c, 0x66, 0x03, 0x03, 0x73, 0x66, 0x7c, 0x00],
|
||||
'H': [0x33, 0x33, 0x33, 0x3f, 0x33, 0x33, 0x33, 0x00],
|
||||
'I': [0x1e, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x1e, 0x00],
|
||||
'J': [0x78, 0x30, 0x30, 0x30, 0x33, 0x33, 0x1e, 0x00],
|
||||
'K': [0x67, 0x66, 0x36, 0x1e, 0x36, 0x66, 0x67, 0x00],
|
||||
'L': [0x0f, 0x06, 0x06, 0x06, 0x46, 0x66, 0x7f, 0x00],
|
||||
'M': [0x63, 0x77, 0x7f, 0x7f, 0x6b, 0x63, 0x63, 0x00],
|
||||
'N': [0x63, 0x67, 0x6f, 0x7b, 0x73, 0x63, 0x63, 0x00],
|
||||
'O': [0x1c, 0x36, 0x63, 0x63, 0x63, 0x36, 0x1c, 0x00],
|
||||
'P': [0x3f, 0x66, 0x66, 0x3e, 0x06, 0x06, 0x0f, 0x00],
|
||||
'Q': [0x1e, 0x33, 0x33, 0x33, 0x3b, 0x1e, 0x38, 0x00],
|
||||
'R': [0x3f, 0x66, 0x66, 0x3e, 0x36, 0x66, 0x67, 0x00],
|
||||
'S': [0x1e, 0x33, 0x07, 0x0e, 0x38, 0x33, 0x1e, 0x00],
|
||||
'T': [0x3f, 0x2d, 0x0c, 0x0c, 0x0c, 0x0c, 0x1e, 0x00],
|
||||
'U': [0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x3f, 0x00],
|
||||
'V': [0x33, 0x33, 0x33, 0x33, 0x33, 0x1e, 0x0c, 0x00],
|
||||
'W': [0x63, 0x63, 0x63, 0x6b, 0x7f, 0x77, 0x63, 0x00],
|
||||
'X': [0x63, 0x63, 0x36, 0x1c, 0x1c, 0x36, 0x63, 0x00],
|
||||
'Y': [0x33, 0x33, 0x33, 0x1e, 0x0c, 0x0c, 0x1e, 0x00],
|
||||
'Z': [0x7f, 0x63, 0x31, 0x18, 0x4c, 0x66, 0x7f, 0x00],
|
||||
'[': [0x1e, 0x06, 0x06, 0x06, 0x06, 0x06, 0x1e, 0x00],
|
||||
'\\': [0x03, 0x06, 0x0c, 0x18, 0x30, 0x60, 0x40, 0x00],
|
||||
']': [0x1e, 0x18, 0x18, 0x18, 0x18, 0x18, 0x1e, 0x00],
|
||||
'^': [0x08, 0x1c, 0x36, 0x63, 0x00, 0x00, 0x00, 0x00],
|
||||
'_': [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff],
|
||||
'`': [0x0c, 0x0c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
'a': [0x00, 0x00, 0x1e, 0x30, 0x3e, 0x33, 0x6e, 0x00],
|
||||
'b': [0x07, 0x06, 0x06, 0x3e, 0x66, 0x66, 0x3b, 0x00],
|
||||
'c': [0x00, 0x00, 0x1e, 0x33, 0x03, 0x33, 0x1e, 0x00],
|
||||
'd': [0x38, 0x30, 0x30, 0x3e, 0x33, 0x33, 0x6e, 0x00],
|
||||
'e': [0x00, 0x00, 0x1e, 0x33, 0x3f, 0x03, 0x1e, 0x00],
|
||||
'f': [0x1c, 0x36, 0x06, 0x0f, 0x06, 0x06, 0x0f, 0x00],
|
||||
'g': [0x00, 0x00, 0x6e, 0x33, 0x33, 0x3e, 0x30, 0x1f],
|
||||
'h': [0x07, 0x06, 0x36, 0x6e, 0x66, 0x66, 0x67, 0x00],
|
||||
'i': [0x0c, 0x00, 0x0e, 0x0c, 0x0c, 0x0c, 0x1e, 0x00],
|
||||
'j': [0x30, 0x00, 0x30, 0x30, 0x30, 0x33, 0x33, 0x1e],
|
||||
'k': [0x07, 0x06, 0x66, 0x36, 0x1e, 0x36, 0x67, 0x00],
|
||||
'l': [0x0e, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x1e, 0x00],
|
||||
'm': [0x00, 0x00, 0x33, 0x7f, 0x7f, 0x6b, 0x63, 0x00],
|
||||
'n': [0x00, 0x00, 0x1f, 0x33, 0x33, 0x33, 0x33, 0x00],
|
||||
'o': [0x00, 0x00, 0x1e, 0x33, 0x33, 0x33, 0x1e, 0x00],
|
||||
'p': [0x00, 0x00, 0x3b, 0x66, 0x66, 0x3e, 0x06, 0x0f],
|
||||
'q': [0x00, 0x00, 0x6e, 0x33, 0x33, 0x3e, 0x30, 0x78],
|
||||
'r': [0x00, 0x00, 0x3b, 0x6e, 0x66, 0x06, 0x0f, 0x00],
|
||||
's': [0x00, 0x00, 0x3e, 0x03, 0x1e, 0x30, 0x1f, 0x00],
|
||||
't': [0x08, 0x0c, 0x3e, 0x0c, 0x0c, 0x2c, 0x18, 0x00],
|
||||
'u': [0x00, 0x00, 0x33, 0x33, 0x33, 0x33, 0x6e, 0x00],
|
||||
'v': [0x00, 0x00, 0x33, 0x33, 0x33, 0x1e, 0x0c, 0x00],
|
||||
'w': [0x00, 0x00, 0x63, 0x6b, 0x7f, 0x7f, 0x36, 0x00],
|
||||
'x': [0x00, 0x00, 0x63, 0x36, 0x1c, 0x36, 0x63, 0x00],
|
||||
'y': [0x00, 0x00, 0x33, 0x33, 0x33, 0x3e, 0x30, 0x1f],
|
||||
'z': [0x00, 0x00, 0x3f, 0x19, 0x0c, 0x26, 0x3f, 0x00],
|
||||
'{': [0x38, 0x0c, 0x0c, 0x07, 0x0c, 0x0c, 0x38, 0x00],
|
||||
'|': [0x18, 0x18, 0x18, 0x00, 0x18, 0x18, 0x18, 0x00],
|
||||
'}': [0x07, 0x0c, 0x0c, 0x38, 0x0c, 0x0c, 0x07, 0x00],
|
||||
'~': [0x6e, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
'·': [0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00], // custom middle-dot separator
|
||||
};
|
||||
|
||||
const BLANK = GLYPHS[' '];
|
||||
|
||||
/** 8 row-bytes for a character; falls back to a blank cell for anything unmapped. */
|
||||
export function glyphFor(ch: string): number[] {
|
||||
return GLYPHS[ch] ?? BLANK;
|
||||
}
|
||||
|
||||
/** Whether a character has a real glyph (used for width estimation / fitting). */
|
||||
export function hasGlyph(ch: string): boolean {
|
||||
return ch in GLYPHS;
|
||||
}
|
||||
|
|
@ -4,15 +4,8 @@
|
|||
*
|
||||
* The NAME list is authoritative (it's the committed baseline set) and is what
|
||||
* lets compare/promote tell an intentional REMOVAL from a flaky/absent render.
|
||||
* The ENGINE tag is best-effort (attributed by scanning which spec writes each
|
||||
* `test-results/<prefix>` and which project runs that spec) and only feeds the
|
||||
* per-engine floors — refine after calibration.
|
||||
*
|
||||
* Engine routing (from the two playwright configs):
|
||||
* e2e/*.spec.ts → chromium-swiftshader (npm run test — wx suite)
|
||||
* kicad/*.spec.ts → firefox-llvmpipe, or chromium-swiftshader if the
|
||||
* spec is in PCBNEW_FAMILY_SPECS (chromium-ci on CI)
|
||||
* web/*.spec.ts → firefox-llvmpipe (web config, --project=firefox)
|
||||
* The ENGINE tag is best-effort (spec-map.ts attributes each name to the spec that
|
||||
* writes it, and that spec's project/engine) and only feeds the per-engine floors.
|
||||
*
|
||||
* CLI (from tests/): tsx tools/screenshots/gen-manifest.ts [--check]
|
||||
* --check exits 1 if the committed manifest is stale (for CI hygiene).
|
||||
|
|
@ -20,58 +13,7 @@
|
|||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { BASELINE_DIRS, MANIFEST_PATH, isIgnored, type Manifest } from './config';
|
||||
|
||||
const CHROMIUM = 'chromium-swiftshader';
|
||||
const FIREFOX = 'firefox-llvmpipe';
|
||||
const DEFAULT_ENGINE = CHROMIUM; // baseline-screenshots is dominated by the wx suite
|
||||
|
||||
function listSpecs(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
if (!fs.existsSync(dir)) return out;
|
||||
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = path.join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...listSpecs(p));
|
||||
// Only spec files — a screenshot literal in a util (e.g. completeWizard's
|
||||
// wizard-*) would be attributed to the util's dir, not its real caller.
|
||||
// Leaving those unmatched lets them fall to the correct chromium default.
|
||||
else if (e.name.endsWith('.spec.ts')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** pcbnew-family spec basenames (routed to chromium-ci on CI), read from the config. */
|
||||
function pcbnewFamily(root: string): Set<string> {
|
||||
const cfg = fs.readFileSync(path.join(root, 'playwright-kicad.config.ts'), 'utf8');
|
||||
const block = cfg.match(/PCBNEW_FAMILY_SPECS\s*=\s*\[([\s\S]*?)\]/)?.[1] ?? '';
|
||||
return new Set([...block.matchAll(/'[^']*?([\w.-]+\.spec\.ts)'/g)].map((m) => m[1]));
|
||||
}
|
||||
|
||||
function engineForSpec(root: string, specPath: string, family: Set<string>): string {
|
||||
const rel = path.relative(root, specPath);
|
||||
const base = path.basename(specPath);
|
||||
if (rel.startsWith('e2e/')) return CHROMIUM;
|
||||
if (rel.startsWith('web/')) return FIREFOX;
|
||||
if (rel.startsWith('kicad/')) return family.has(base) ? CHROMIUM : FIREFOX;
|
||||
return DEFAULT_ENGINE;
|
||||
}
|
||||
|
||||
/** Build prefix → engine from every `test-results/<prefix>` literal in the specs. */
|
||||
function prefixEngineMap(root: string, family: Set<string>): Array<{ prefix: string; engine: string }> {
|
||||
const map = new Map<string, string>();
|
||||
for (const dir of ['e2e', 'kicad', 'web']) {
|
||||
for (const spec of listSpecs(path.join(root, dir))) {
|
||||
const engine = engineForSpec(root, spec, family);
|
||||
const content = fs.readFileSync(spec, 'utf8');
|
||||
for (const m of content.matchAll(/test-results\/([A-Za-z0-9_-]+)/g)) {
|
||||
const prefix = m[1];
|
||||
// First writer wins; a chromium spec shouldn't be overridden by a later firefox one for the same literal.
|
||||
if (!map.has(prefix)) map.set(prefix, engine);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Longest prefix first so the most specific match wins.
|
||||
return [...map.entries()].map(([prefix, engine]) => ({ prefix, engine })).sort((a, b) => b.prefix.length - a.prefix.length);
|
||||
}
|
||||
import { buildSpecResolver } from './spec-map';
|
||||
|
||||
function listBaselines(root: string): string[] {
|
||||
const names = new Set<string>();
|
||||
|
|
@ -86,19 +28,16 @@ function listBaselines(root: string): string[] {
|
|||
function main(): void {
|
||||
const check = process.argv.includes('--check');
|
||||
const root = process.cwd();
|
||||
const family = pcbnewFamily(root);
|
||||
const prefixes = prefixEngineMap(root, family);
|
||||
const resolver = buildSpecResolver(root);
|
||||
|
||||
let unmatched = 0;
|
||||
const screenshots = listBaselines(root).map((name) => {
|
||||
const stem = name.replace(/\.png$/i, '');
|
||||
const hit = prefixes.find((p) => stem === p.prefix || stem.startsWith(p.prefix));
|
||||
if (!hit) unmatched++;
|
||||
return { name, engine: hit?.engine ?? DEFAULT_ENGINE };
|
||||
if (resolver.specFor(name) === null) unmatched++;
|
||||
return { name, engine: resolver.engineFor(name) };
|
||||
});
|
||||
|
||||
const manifest: Manifest & { _note: string } = {
|
||||
_note: 'engine tags are best-effort (gen-manifest.ts); the name list is authoritative. Refine engines after calibration.',
|
||||
_note: 'engine tags are best-effort (spec-map.ts); the name list is authoritative. Refine engines after calibration.',
|
||||
screenshots,
|
||||
};
|
||||
const json = JSON.stringify(manifest, null, 2) + '\n';
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@
|
|||
import * as fs from 'fs';
|
||||
import { PNG } from 'pngjs';
|
||||
import pixelmatch from 'pixelmatch';
|
||||
import { PIXELMATCH, DIFF_COLOR, CLUSTER, TRIPTYCH } from './config';
|
||||
import { PIXELMATCH, DIFF_COLOR, CLUSTER, TRIPTYCH, LABEL } from './config';
|
||||
import { glyphFor } from './font8x8';
|
||||
|
||||
export type Box = { x: number; y: number; width: number; height: number; area: number };
|
||||
|
||||
|
|
@ -254,3 +255,62 @@ export function composite(panels: PNG[]): PNG {
|
|||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Draw `text` at (x0,y0) with the 8x8 bitmap font, each pixel scaled `scale`×. Clips at edges. */
|
||||
export function drawText(png: PNG, x0: number, y0: number, text: string, color: [number, number, number], scale: number): void {
|
||||
let x = x0;
|
||||
for (const ch of text) {
|
||||
const glyph = glyphFor(ch);
|
||||
for (let row = 0; row < 8; row++) {
|
||||
const bits = glyph[row];
|
||||
if (!bits) continue;
|
||||
for (let col = 0; col < 8; col++) {
|
||||
if (!((bits >> col) & 1)) continue;
|
||||
for (let dy = 0; dy < scale; dy++) {
|
||||
const py = y0 + row * scale + dy;
|
||||
if (py < 0 || py >= png.height) continue;
|
||||
for (let dx = 0; dx < scale; dx++) {
|
||||
const px = x + col * scale + dx;
|
||||
if (px < 0 || px >= png.width) continue;
|
||||
const o = (py * png.width + px) * 4;
|
||||
png.data[o] = color[0];
|
||||
png.data[o + 1] = color[1];
|
||||
png.data[o + 2] = color[2];
|
||||
png.data[o + 3] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
x += 9 * scale; // 8px glyph + 1px spacing
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a copy of `png` with a `bg`-coloured caption strip appended at the bottom,
|
||||
* showing `text` in white. Auto-fits the font scale to the width; if even scale 1
|
||||
* overflows, truncates the tail (keeps the name, trims the spec) with `..`.
|
||||
*/
|
||||
export function withBottomLabel(png: PNG, text: string, bg: [number, number, number]): PNG {
|
||||
const advance = 9; // per-char glyph cells at scale 1
|
||||
const maxW = Math.max(1, png.width - 2 * LABEL.hpad);
|
||||
let scale = LABEL.maxScale;
|
||||
while (scale > 1 && text.length * advance * scale > maxW) scale--;
|
||||
let label = text;
|
||||
const maxChars = Math.max(1, Math.floor(maxW / (advance * scale)));
|
||||
if (label.length > maxChars) label = label.slice(0, Math.max(1, maxChars - 2)) + '..';
|
||||
|
||||
const stripH = 8 * scale + 2 * LABEL.vpad;
|
||||
const out = new PNG({ width: png.width, height: png.height + stripH });
|
||||
png.data.copy(out.data, 0, 0, png.data.length); // original image on top (same width)
|
||||
for (let y = png.height; y < out.height; y++) {
|
||||
for (let x = 0; x < out.width; x++) {
|
||||
const o = (y * out.width + x) * 4;
|
||||
out.data[o] = bg[0];
|
||||
out.data[o + 1] = bg[1];
|
||||
out.data[o + 2] = bg[2];
|
||||
out.data[o + 3] = 255;
|
||||
}
|
||||
}
|
||||
drawText(out, LABEL.hpad, png.height + LABEL.vpad, label, LABEL.text, scale);
|
||||
return out;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -120,8 +120,17 @@ export function buildAttachments(root: string, report: Report | null): { files:
|
|||
const a = attach(root, ad.image, `ADDED_${ad.name}`);
|
||||
if (a) files.push(a);
|
||||
}
|
||||
const removedToShow = report.removed.length > FLOOD_N ? report.removed.slice(0, 3) : report.removed;
|
||||
if (report.removed.length > FLOOD_N) {
|
||||
notes.push(`➖ ${report.removed.length} removed (showing ${removedToShow.length})`);
|
||||
}
|
||||
for (const rm of removedToShow) {
|
||||
if (files.length >= MAX_TOTAL_FILES) break;
|
||||
const a = attach(root, rm.image, `REMOVED_${rm.name}`);
|
||||
if (a) files.push(a);
|
||||
}
|
||||
const shown = files.length;
|
||||
const wanted = report.changed.length + addedToShow.length;
|
||||
const wanted = report.changed.length + addedToShow.length + removedToShow.length;
|
||||
if (wanted > shown) notes.push(`(${wanted - shown} more images omitted — see the CI artifact)`);
|
||||
return { files, notes };
|
||||
}
|
||||
|
|
|
|||
81
tests/tools/screenshots/spec-map.ts
Normal file
81
tests/tools/screenshots/spec-map.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* Resolve a screenshot name → the spec that writes it (and the engine that renders
|
||||
* it), by scanning the specs for `page.screenshot({ path: 'test-results/<prefix>…' })`
|
||||
* literals. Shared by gen-manifest.ts (engine tags) and compare/changelog (caption
|
||||
* labels).
|
||||
*
|
||||
* Attribution is longest-prefix (`pcbnew-loaded` → the spec that writes `pcbnew-…`).
|
||||
* Names written only by a helper (e.g. `wizard-*` via completeWizard, which lives in
|
||||
* a util not a *.spec.ts) don't match → specFor returns null (the caption then just
|
||||
* shows the name).
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export const CHROMIUM = 'chromium-swiftshader';
|
||||
export const FIREFOX = 'firefox-llvmpipe';
|
||||
export const DEFAULT_ENGINE = CHROMIUM; // baseline-screenshots is dominated by the wx suite
|
||||
|
||||
function listSpecs(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
if (!fs.existsSync(dir)) return out;
|
||||
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = path.join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...listSpecs(p));
|
||||
// Only *.spec.ts — a literal in a util would be mis-attributed to the util's dir.
|
||||
else if (e.name.endsWith('.spec.ts')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** pcbnew-family spec basenames (routed to chromium-ci on CI), read from the config. */
|
||||
function pcbnewFamily(root: string): Set<string> {
|
||||
const cfg = fs.readFileSync(path.join(root, 'playwright-kicad.config.ts'), 'utf8');
|
||||
const block = cfg.match(/PCBNEW_FAMILY_SPECS\s*=\s*\[([\s\S]*?)\]/)?.[1] ?? '';
|
||||
return new Set([...block.matchAll(/'[^']*?([\w.-]+\.spec\.ts)'/g)].map((m) => m[1]));
|
||||
}
|
||||
|
||||
function engineForSpec(root: string, specPath: string, family: Set<string>): string {
|
||||
const rel = path.relative(root, specPath);
|
||||
const base = path.basename(specPath);
|
||||
if (rel.startsWith('e2e/')) return CHROMIUM;
|
||||
if (rel.startsWith('web/')) return FIREFOX;
|
||||
if (rel.startsWith('kicad/')) return family.has(base) ? CHROMIUM : FIREFOX;
|
||||
return DEFAULT_ENGINE;
|
||||
}
|
||||
|
||||
type PrefixEntry = { prefix: string; engine: string; spec: string };
|
||||
|
||||
/** prefix → {engine, spec-rel-to-tests}, longest-prefix first. */
|
||||
function prefixMap(root: string, family: Set<string>): PrefixEntry[] {
|
||||
const map = new Map<string, { engine: string; spec: string }>();
|
||||
for (const dir of ['e2e', 'kicad', 'web']) {
|
||||
for (const spec of listSpecs(path.join(root, dir))) {
|
||||
const engine = engineForSpec(root, spec, family);
|
||||
const rel = path.relative(root, spec); // e.g. 'kicad/pcbnew.spec.ts'
|
||||
const content = fs.readFileSync(spec, 'utf8');
|
||||
for (const m of content.matchAll(/test-results\/([A-Za-z0-9_-]+)/g)) {
|
||||
const prefix = m[1];
|
||||
if (!map.has(prefix)) map.set(prefix, { engine, spec: rel }); // first writer wins
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...map.entries()]
|
||||
.map(([prefix, v]) => ({ prefix, ...v }))
|
||||
.sort((a, b) => b.prefix.length - a.prefix.length);
|
||||
}
|
||||
|
||||
export type SpecResolver = { specFor(name: string): string | null; engineFor(name: string): string };
|
||||
|
||||
/** Build a name→spec/engine resolver by scanning the specs under `root` once. */
|
||||
export function buildSpecResolver(root: string): SpecResolver {
|
||||
const prefixes = prefixMap(root, pcbnewFamily(root));
|
||||
const match = (name: string): PrefixEntry | undefined => {
|
||||
const stem = name.replace(/\.png$/i, '');
|
||||
return prefixes.find((p) => stem === p.prefix || stem.startsWith(p.prefix));
|
||||
};
|
||||
return {
|
||||
specFor: (name) => match(name)?.spec ?? null,
|
||||
engineFor: (name) => match(name)?.engine ?? DEFAULT_ENGINE,
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue