AI_Assistant/pcbjam
Watch
1
0
Fork
You've already forked pcbjam
 0
Code Issues Pull requests Projects Releases 7 Packages Wiki Activity Actions

feat(tests): caption posted screenshots (name + spec); raise drift floor to 0.5%

Browse source 
Bake a caption strip onto every posted screenshot composite — status + name +
the spec that produces it — for changed, added, and removed; removed now shows
the old baseline image (was a text-only line). Zero native-dep: an embedded
public-domain 8x8 bitmap font (font8x8.ts) rendered by image-ops `withBottomLabel`;
the name→spec attribution is factored out of gen-manifest into a shared
`spec-map.ts` resolver. Bottom strip, colour per status (green/red/orange).
Applies in both compare (drift gate) and changelog (git-history diff), and
post-discord now attaches the captioned removed images.

Also raise the per-engine drift floor 0.2% → 0.5% (changedRatio) to absorb the
sub-1% inter-run flakiness seen after the re-baseline, while still catching real
localized changes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Vaczi 2026-07-02 12:07:41 +02:00
commit 9787efc2c9
9 changed files with 360 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

19
tests/tools/screenshots/compare.ts
View file

@ -24,8 +24,11 @@ import {
type Manifest,
floorFor,
isIgnored,
labelText,
LABEL,
} from './config';
import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, type Box } from './image-ops';
import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, withBottomLabel, type Box } from './image-ops';
import { buildSpecResolver } from './spec-map';
export type PairVerdict = 'unchanged' | 'changed';
@ -79,7 +82,7 @@ export type Report = {
generatedFor: string | null;
changed: ChangedEntry[];
added: Array<{ name: string; image: string }>;
removed: Array<{ name: string }>;
removed: Array<{ name: string; image: string }>;
unchangedCount: number;
/** many changes, mostly drift-like ⇒ probably a host Mesa/font refresh; re-promote rather than debug */
driftLikely: boolean;
@ -130,6 +133,7 @@ export function classify(root: string, sha: string | null): Report {
const manifest = loadManifest(root);
const outDir = path.join(root, DIFF_OUT_DIR);
fs.mkdirSync(outDir, { recursive: true });
const { specFor } = buildSpecResolver(root); // name → spec, for the caption strip
const report: Report = {
generatedFor: sha,
@ -148,7 +152,11 @@ export function classify(root: string, sha: string | null): Report {
// an intentional removal. (The stronger "did the spec actually run" cross-check
// against the Playwright JSON report lands with the manifest work.)
if (manifest?.screenshots.some((e) => e.name === name)) {
report.removed.push({ name });
// Removed now gets a captioned image (the old baseline) so it's visible in Discord.
const imageRel = path.join(DIFF_OUT_DIR, `${name}.removed.png`);
const labeled = withBottomLabel(loadPng(baselinePath), labelText('removed', name, specFor(name)), LABEL.colors.removed);
savePng(path.join(root, imageRel), labeled);
report.removed.push({ name, image: imageRel });
}
continue;
}
@ -164,7 +172,7 @@ export function classify(root: string, sha: string | null): Report {
}
const triptychRel = path.join(DIFF_OUT_DIR, `${name}.triptych.png`);
const heatmapRel = path.join(DIFF_OUT_DIR, `${name}.heatmap.png`);
savePng(path.join(root, triptychRel), triptych);
savePng(path.join(root, triptychRel), withBottomLabel(triptych, labelText('changed', name, specFor(name)), LABEL.colors.changed));
savePng(path.join(root, heatmapRel), heatmap);
report.changed.push({ ...result, triptych: triptychRel, heatmap: heatmapRel });
}
@ -173,7 +181,8 @@ export function classify(root: string, sha: string | null): Report {
for (const name of actuals) {
if (baselines.has(name)) continue;
const imageRel = path.join(DIFF_OUT_DIR, `${name}.added.png`);
fs.copyFileSync(path.join(resultsDir, name), path.join(root, imageRel));
const labeled = withBottomLabel(loadPng(path.join(resultsDir, name)), labelText('added', name, specFor(name)), LABEL.colors.added);
savePng(path.join(root, imageRel), labeled);
report.added.push({ name, image: imageRel });
}