AI_Assistant/pcbjam
Watch
1
0
Fork
You've already forked pcbjam
 0
Code Issues Pull requests Projects Releases 7 Packages Wiki Activity Actions

feat(tests): caption posted screenshots (name + spec); raise drift floor to 0.5%

Browse source 
Bake a caption strip onto every posted screenshot composite — status + name +
the spec that produces it — for changed, added, and removed; removed now shows
the old baseline image (was a text-only line). Zero native-dep: an embedded
public-domain 8x8 bitmap font (font8x8.ts) rendered by image-ops `withBottomLabel`;
the name→spec attribution is factored out of gen-manifest into a shared
`spec-map.ts` resolver. Bottom strip, colour per status (green/red/orange).
Applies in both compare (drift gate) and changelog (git-history diff), and
post-discord now attaches the captioned removed images.

Also raise the per-engine drift floor 0.2% → 0.5% (changedRatio) to absorb the
sub-1% inter-run flakiness seen after the re-baseline, while still catching real
localized changes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Vaczi 2026-07-02 12:07:41 +02:00
commit 9787efc2c9
9 changed files with 360 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

75
tests/tools/screenshots/gen-manifest.ts
View file

@ -4,15 +4,8 @@
*
* The NAME list is authoritative (it's the committed baseline set) and is what
* lets compare/promote tell an intentional REMOVAL from a flaky/absent render.
* The ENGINE tag is best-effort (attributed by scanning which spec writes each
* `test-results/<prefix>` and which project runs that spec) and only feeds the
* per-engine floors — refine after calibration.
*
* Engine routing (from the two playwright configs):
* e2e/*.spec.ts → chromium-swiftshader (npm run test — wx suite)
* kicad/*.spec.ts → firefox-llvmpipe, or chromium-swiftshader if the
* spec is in PCBNEW_FAMILY_SPECS (chromium-ci on CI)
* web/*.spec.ts → firefox-llvmpipe (web config, --project=firefox)
* The ENGINE tag is best-effort (spec-map.ts attributes each name to the spec that
* writes it, and that spec's project/engine) and only feeds the per-engine floors.
*
* CLI (from tests/): tsx tools/screenshots/gen-manifest.ts [--check]
* --check exits 1 if the committed manifest is stale (for CI hygiene).
@ -20,58 +13,7 @@
import * as fs from 'fs';
import * as path from 'path';
import { BASELINE_DIRS, MANIFEST_PATH, isIgnored, type Manifest } from './config';
const CHROMIUM = 'chromium-swiftshader';
const FIREFOX = 'firefox-llvmpipe';
const DEFAULT_ENGINE = CHROMIUM; // baseline-screenshots is dominated by the wx suite
function listSpecs(dir: string): string[] {
const out: string[] = [];
if (!fs.existsSync(dir)) return out;
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
const p = path.join(dir, e.name);
if (e.isDirectory()) out.push(...listSpecs(p));
// Only spec files — a screenshot literal in a util (e.g. completeWizard's
// wizard-*) would be attributed to the util's dir, not its real caller.
// Leaving those unmatched lets them fall to the correct chromium default.
else if (e.name.endsWith('.spec.ts')) out.push(p);
}
return out;
}
/** pcbnew-family spec basenames (routed to chromium-ci on CI), read from the config. */
function pcbnewFamily(root: string): Set<string> {
const cfg = fs.readFileSync(path.join(root, 'playwright-kicad.config.ts'), 'utf8');
const block = cfg.match(/PCBNEW_FAMILY_SPECS\s*=\s*\[([\s\S]*?)\]/)?.[1] ?? '';
return new Set([...block.matchAll(/'[^']*?([\w.-]+\.spec\.ts)'/g)].map((m) => m[1]));
}
function engineForSpec(root: string, specPath: string, family: Set<string>): string {
const rel = path.relative(root, specPath);
const base = path.basename(specPath);
if (rel.startsWith('e2e/')) return CHROMIUM;
if (rel.startsWith('web/')) return FIREFOX;
if (rel.startsWith('kicad/')) return family.has(base) ? CHROMIUM : FIREFOX;
return DEFAULT_ENGINE;
}
/** Build prefix → engine from every `test-results/<prefix>` literal in the specs. */
function prefixEngineMap(root: string, family: Set<string>): Array<{ prefix: string; engine: string }> {
const map = new Map<string, string>();
for (const dir of ['e2e', 'kicad', 'web']) {
for (const spec of listSpecs(path.join(root, dir))) {
const engine = engineForSpec(root, spec, family);
const content = fs.readFileSync(spec, 'utf8');
for (const m of content.matchAll(/test-results\/([A-Za-z0-9_-]+)/g)) {
const prefix = m[1];
// First writer wins; a chromium spec shouldn't be overridden by a later firefox one for the same literal.
if (!map.has(prefix)) map.set(prefix, engine);
}
}
}
// Longest prefix first so the most specific match wins.
return [...map.entries()].map(([prefix, engine]) => ({ prefix, engine })).sort((a, b) => b.prefix.length - a.prefix.length);
}
import { buildSpecResolver } from './spec-map';
function listBaselines(root: string): string[] {
const names = new Set<string>();
@ -86,19 +28,16 @@ function listBaselines(root: string): string[] {
function main(): void {
const check = process.argv.includes('--check');
const root = process.cwd();
const family = pcbnewFamily(root);
const prefixes = prefixEngineMap(root, family);
const resolver = buildSpecResolver(root);
let unmatched = 0;
const screenshots = listBaselines(root).map((name) => {
const stem = name.replace(/\.png$/i, '');
const hit = prefixes.find((p) => stem === p.prefix || stem.startsWith(p.prefix));
if (!hit) unmatched++;
return { name, engine: hit?.engine ?? DEFAULT_ENGINE };
if (resolver.specFor(name) === null) unmatched++;
return { name, engine: resolver.engineFor(name) };
});
const manifest: Manifest & { _note: string } = {
_note: 'engine tags are best-effort (gen-manifest.ts); the name list is authoritative. Refine engines after calibration.',
_note: 'engine tags are best-effort (spec-map.ts); the name list is authoritative. Refine engines after calibration.',
screenshots,
};
const json = JSON.stringify(manifest, null, 2) + '\n';