pcbjam/tests/tools/screenshots/post-discord.ts
Viktor Vaczi 13551f4f22 feat(tests): screenshot regression + Discord review, perf-tracked
New tooling in tests/tools/screenshots/ (TypeScript via tsx):
- compare.ts: one pixelmatch engine (AA-excluded), connected-component
  "where to look" boxes, old|new+boxes|heatmap triptych, per-engine floors.
- promote.ts: churn-free updater — overwrite a baseline only when decoded
  pixels differ beyond the floor, copying CI bytes verbatim (no re-encode
  churn); pulls a CI run via `gh run download` or a local --from dir.
- post-discord.ts: always-on CI-on-main report (SHA + e2e status + the
  track-only runtime-perf table), then screenshot triptychs, batched +
  size-capped + flood-collapsed + 429-aware.
- perf-report.ts: perf table with Δ vs the previous main run (via gh).
- changelog.ts: no-build git-history baseline differ (Discord trigger B).
- noise.ts / gen-manifest.ts: calibration + manifest generation.

CI wiring:
- wasm-build.yml: post-test step runs the gate + report on the already-
  produced test-results (no extra build); report-only (continue-on-error),
  posts only on push to main, inert without DISCORD_WEBHOOK_URL.
- ci-ubicloud.yml: secrets: inherit (pass the webhook through).
- screenshot-changelog.yml: ~30s no-build changelog on baseline changes.

screenshot-manifest.json: canonical 354-name set + best-effort engine tags
(313 chromium-swiftshader / 41 firefox-llvmpipe).

Normalize scale:'device'->'css' across 18 spec files (no-op at CI DSF=1)
so committed baselines are uniformly css-scaled.

Design: CI's Linux render is the single source of truth; no pinned
container (accept rare env drift -> re-promote); dev commits via promote.
Replaces the byte-cmp compare-screenshots.sh + file-size-proxy
update-baseline-screenshots.sh (kept for now until the first re-baseline).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-01 18:17:20 +02:00

226 lines
9.2 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* The always-on CI-on-main Discord report.
*
* One message leads with the commit SHA + e2e pass/fail + the runtime-perf table
* (perf-report.ts), then screenshot changes are attached as triptychs
* (old | new+boxes | heatmap), ADDED images, and a REMOVED list. Because perf is
* always present, the comment fires on every main commit even with no screenshot
* change — "see it in hindsight on every commit."
*
* Safety / robustness:
* - Posts only on push to main (unless --force); a missing DISCORD_WEBHOOK_URL is
* a silent no-op, so it's inert on PRs / forks (which can't see the secret).
* - Attachments batched ≤10/message and downscaled to fit Discord's size caps;
* 429 Retry-After honoured; first-run flood (hundreds of ADDED) collapsed.
* - --dry-run composes everything and prints it without POSTing (how it's tested).
*
* CLI (from tests/):
* tsx tools/screenshots/post-discord.ts [--dry-run] [--force] [--e2e pass|fail] [--subject S]
*/
import * as fs from 'fs';
import * as path from 'path';
import { execFileSync } from 'child_process';
import { DIFF_OUT_DIR } from './config';
import { loadPng, encodeWithinCap } from './image-ops';
import { buildPerfReport, fetchPreviousPerf } from './perf-report';
import type { Report } from './compare';
const MAX_FILE_BYTES = 8 * 1024 * 1024;
const MAX_MSG_BYTES = 24 * 1024 * 1024;
const MAX_FILES_PER_MSG = 10;
const MAX_TOTAL_FILES = 30; // ~3 messages of images; excess is summarized, not posted
const FLOOD_N = 12; // more ADDED than this ⇒ show a few exemplars, not all
type Attachment = { name: string; buffer: Buffer };
type Message = { content: string; files: Attachment[] };
function sanitize(name: string): string {
return name.replace(/[^\w.-]+/g, '_');
}
function e2eBadge(status: string | undefined): string {
if (status === 'pass') return '✅ e2e passed';
if (status === 'fail') return '❌ e2e failed';
return ' e2e status unknown';
}
function commitSubject(sha: string | undefined): string {
if (!sha) return '';
try {
return execFileSync('git', ['log', '-1', '--pretty=%s', sha], { encoding: 'utf8' }).trim();
} catch {
return '';
}
}
function readReport(root: string): Report | null {
const p = path.join(root, DIFF_OUT_DIR, 'report.json');
if (!fs.existsSync(p)) return null;
try {
return JSON.parse(fs.readFileSync(p, 'utf8')) as Report;
} catch {
return null;
}
}
/** Encode a PNG path to a within-cap attachment, or null if missing/unreadable. */
function attach(root: string, rel: string, name: string): Attachment | null {
const abs = path.isAbsolute(rel) ? rel : path.join(root, rel);
if (!fs.existsSync(abs)) return null;
try {
return { name: sanitize(name), buffer: encodeWithinCap(loadPng(abs), MAX_FILE_BYTES) };
} catch {
return null;
}
}
/** Build the header text: SHA, subject, e2e, perf table, screenshot summary, removed list. */
function buildHeader(report: Report | null, perfBlock: string, meta: { sha?: string; subject: string; e2e?: string }): string {
const shortSha = meta.sha ? meta.sha.slice(0, 7) : 'local';
const lines = [`📸 **CI screenshot + perf report** · \`${shortSha}\``];
if (meta.subject) lines.push(`> ${meta.subject}`);
lines.push(e2eBadge(meta.e2e));
if (perfBlock) lines.push('', perfBlock);
lines.push('');
if (!report || (!report.changed.length && !report.added.length && !report.removed.length)) {
lines.push('✅ no screenshot drift');
} else if (report.driftLikely) {
lines.push(`⚠️ **${report.changed.length} screenshots changed broadly** — looks like host env drift → re-promote (\`npm run screenshots:promote\`)`);
} else {
lines.push(`⚠️ **screenshot drift**: ${report.changed.length} changed, ${report.added.length} added, ${report.removed.length} removed`);
}
if (report?.removed.length) {
lines.push(' REMOVED: ' + report.removed.map((r) => `\`${r.name}\``).join(', '));
}
return lines.join('\n');
}
/** Collect the image attachments (changed triptychs + added images) with flood-collapse + total cap. */
export function buildAttachments(root: string, report: Report | null): { files: Attachment[]; notes: string[] } {
const files: Attachment[] = [];
const notes: string[] = [];
if (!report) return { files, notes };
for (const c of report.changed) {
if (files.length >= MAX_TOTAL_FILES) break;
const a = attach(root, c.triptych, `CHANGED_${c.name}`);
if (a) files.push(a);
}
const addedToShow = report.added.length > FLOOD_N ? report.added.slice(0, 3) : report.added;
if (report.added.length > FLOOD_N) {
notes.push(` ${report.added.length} added (showing ${addedToShow.length})`);
}
for (const ad of addedToShow) {
if (files.length >= MAX_TOTAL_FILES) break;
const a = attach(root, ad.image, `ADDED_${ad.name}`);
if (a) files.push(a);
}
const shown = files.length;
const wanted = report.changed.length + addedToShow.length;
if (wanted > shown) notes.push(`(${wanted - shown} more images omitted — see the CI artifact)`);
return { files, notes };
}
/** Split attachments into messages of ≤10 files and ≤MAX_MSG_BYTES; content only on the first. */
export function paginate(header: string, files: Attachment[]): Message[] {
if (!files.length) return [{ content: header, files: [] }];
const messages: Message[] = [];
let batch: Attachment[] = [];
let bytes = 0;
const flush = () => {
messages.push({ content: messages.length === 0 ? header : '', files: batch });
batch = [];
bytes = 0;
};
for (const f of files) {
if (batch.length >= MAX_FILES_PER_MSG || bytes + f.buffer.length > MAX_MSG_BYTES) flush();
batch.push(f);
bytes += f.buffer.length;
}
if (batch.length || messages.length === 0) flush();
return messages;
}
async function sleep(ms: number): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}
export async function postMessage(webhook: string, msg: Message): Promise<void> {
for (let attemptNo = 0; attemptNo < 6; attemptNo++) {
const form = new FormData();
form.append('payload_json', JSON.stringify({ content: msg.content || '', allowed_mentions: { parse: [] } }));
msg.files.forEach((f, i) => form.append(`files[${i}]`, new Blob([new Uint8Array(f.buffer)], { type: 'image/png' }), f.name));
const res = await fetch(webhook, { method: 'POST', body: form });
if (res.ok) return;
if (res.status === 429) {
const retryAfter = Number(res.headers.get('retry-after')) || 2;
await sleep((retryAfter + 0.5) * 1000);
continue;
}
throw new Error(`Discord POST failed ${res.status}: ${(await res.text()).slice(0, 300)}`);
}
throw new Error('Discord POST failed after retries (429)');
}
function parseArgs(argv: string[]): Record<string, string | boolean> {
const out: Record<string, string | boolean> = {};
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === '--dry-run') out.dryRun = true;
else if (a === '--force') out.force = true;
else if (a === '--e2e') out.e2e = argv[++i];
else if (a === '--subject') out.subject = argv[++i];
else if (a === '--repo') out.repo = argv[++i];
}
return out;
}
async function main(): Promise<void> {
const args = parseArgs(process.argv.slice(2));
const root = process.cwd();
const isMainPush = process.env.GITHUB_REF === 'refs/heads/main' && process.env.GITHUB_EVENT_NAME === 'push';
if (!args.force && !args.dryRun && !isMainPush) {
console.log('[discord] not a push to main — skipping');
return;
}
const webhook = process.env.DISCORD_WEBHOOK_URL;
if (!webhook && !args.dryRun) {
console.log('[discord] DISCORD_WEBHOOK_URL unset — skipping (inert on PRs/forks)');
return;
}
const sha = process.env.GITHUB_SHA;
const report = readReport(root);
const prevDir = fetchPreviousPerf((args.repo as string) || process.env.GITHUB_REPOSITORY, sha);
const { block: perfBlock } = buildPerfReport({ prevDir });
let header = buildHeader(report, perfBlock, {
sha,
subject: (args.subject as string) ?? commitSubject(sha),
e2e: args.e2e as string,
});
const { files, notes } = buildAttachments(root, report);
if (notes.length) header += '\n' + notes.join('\n');
const messages = paginate(header, files);
if (args.dryRun) {
for (const [i, m] of messages.entries()) {
console.log(`--- message ${i + 1}/${messages.length} (${m.files.length} files, ${m.files.reduce((s, f) => s + f.buffer.length, 0)} bytes) ---`);
if (m.content) console.log(m.content);
for (const f of m.files) console.log(` [attach] ${f.name} (${f.buffer.length} bytes)`);
}
return;
}
for (const m of messages) await postMessage(webhook!, m);
console.log(`[discord] posted ${messages.length} message(s)`);
}
if (require.main === module) {
main().catch((e) => {
console.error(`[discord] ${e.message}`);
process.exitCode = 1;
});
}