feat(tests): caption posted screenshots (name + spec); raise drift floor to 0.5%
Bake a caption strip onto every posted screenshot composite — status + name + the spec that produces it — for changed, added, and removed; removed now shows the old baseline image (was a text-only line). Zero native-dep: an embedded public-domain 8x8 bitmap font (font8x8.ts) rendered by image-ops `withBottomLabel`; the name→spec attribution is factored out of gen-manifest into a shared `spec-map.ts` resolver. Bottom strip, colour per status (green/red/orange). Applies in both compare (drift gate) and changelog (git-history diff), and post-discord now attaches the captioned removed images. Also raise the per-engine drift floor 0.2% → 0.5% (changedRatio) to absorb the sub-1% inter-run flakiness seen after the re-baseline, while still catching real localized changes. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
4330c9344e
commit
9787efc2c9
9 changed files with 360 additions and 101 deletions
81
tests/tools/screenshots/spec-map.ts
Normal file
81
tests/tools/screenshots/spec-map.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* Resolve a screenshot name → the spec that writes it (and the engine that renders
|
||||
* it), by scanning the specs for `page.screenshot({ path: 'test-results/<prefix>…' })`
|
||||
* literals. Shared by gen-manifest.ts (engine tags) and compare/changelog (caption
|
||||
* labels).
|
||||
*
|
||||
* Attribution is longest-prefix (`pcbnew-loaded` → the spec that writes `pcbnew-…`).
|
||||
* Names written only by a helper (e.g. `wizard-*` via completeWizard, which lives in
|
||||
* a util not a *.spec.ts) don't match → specFor returns null (the caption then just
|
||||
* shows the name).
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export const CHROMIUM = 'chromium-swiftshader';
|
||||
export const FIREFOX = 'firefox-llvmpipe';
|
||||
export const DEFAULT_ENGINE = CHROMIUM; // baseline-screenshots is dominated by the wx suite
|
||||
|
||||
function listSpecs(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
if (!fs.existsSync(dir)) return out;
|
||||
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = path.join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...listSpecs(p));
|
||||
// Only *.spec.ts — a literal in a util would be mis-attributed to the util's dir.
|
||||
else if (e.name.endsWith('.spec.ts')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** pcbnew-family spec basenames (routed to chromium-ci on CI), read from the config. */
|
||||
function pcbnewFamily(root: string): Set<string> {
|
||||
const cfg = fs.readFileSync(path.join(root, 'playwright-kicad.config.ts'), 'utf8');
|
||||
const block = cfg.match(/PCBNEW_FAMILY_SPECS\s*=\s*\[([\s\S]*?)\]/)?.[1] ?? '';
|
||||
return new Set([...block.matchAll(/'[^']*?([\w.-]+\.spec\.ts)'/g)].map((m) => m[1]));
|
||||
}
|
||||
|
||||
function engineForSpec(root: string, specPath: string, family: Set<string>): string {
|
||||
const rel = path.relative(root, specPath);
|
||||
const base = path.basename(specPath);
|
||||
if (rel.startsWith('e2e/')) return CHROMIUM;
|
||||
if (rel.startsWith('web/')) return FIREFOX;
|
||||
if (rel.startsWith('kicad/')) return family.has(base) ? CHROMIUM : FIREFOX;
|
||||
return DEFAULT_ENGINE;
|
||||
}
|
||||
|
||||
type PrefixEntry = { prefix: string; engine: string; spec: string };
|
||||
|
||||
/** prefix → {engine, spec-rel-to-tests}, longest-prefix first. */
|
||||
function prefixMap(root: string, family: Set<string>): PrefixEntry[] {
|
||||
const map = new Map<string, { engine: string; spec: string }>();
|
||||
for (const dir of ['e2e', 'kicad', 'web']) {
|
||||
for (const spec of listSpecs(path.join(root, dir))) {
|
||||
const engine = engineForSpec(root, spec, family);
|
||||
const rel = path.relative(root, spec); // e.g. 'kicad/pcbnew.spec.ts'
|
||||
const content = fs.readFileSync(spec, 'utf8');
|
||||
for (const m of content.matchAll(/test-results\/([A-Za-z0-9_-]+)/g)) {
|
||||
const prefix = m[1];
|
||||
if (!map.has(prefix)) map.set(prefix, { engine, spec: rel }); // first writer wins
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...map.entries()]
|
||||
.map(([prefix, v]) => ({ prefix, ...v }))
|
||||
.sort((a, b) => b.prefix.length - a.prefix.length);
|
||||
}
|
||||
|
||||
export type SpecResolver = { specFor(name: string): string | null; engineFor(name: string): string };
|
||||
|
||||
/** Build a name→spec/engine resolver by scanning the specs under `root` once. */
|
||||
export function buildSpecResolver(root: string): SpecResolver {
|
||||
const prefixes = prefixMap(root, pcbnewFamily(root));
|
||||
const match = (name: string): PrefixEntry | undefined => {
|
||||
const stem = name.replace(/\.png$/i, '');
|
||||
return prefixes.find((p) => stem === p.prefix || stem.startsWith(p.prefix));
|
||||
};
|
||||
return {
|
||||
specFor: (name) => match(name)?.spec ?? null,
|
||||
engineFor: (name) => match(name)?.engine ?? DEFAULT_ENGINE,
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue