fix(tests): enforce screenshot gate; drop shadow dir, flaky retinascale, 12 stale
Finalize the screenshot review system after the first Linux re-baseline (that run showed 355/356 stable — the placeholder floor is fine as-is): - ENFORCE the gate: `screenshots:check --fail-on-change` now fails the build on any changed/added/removed vs baselines (it still posts the drift report first, so the failure is actionable; a real render change → re-promote). The generic "CI failed" notice is suppressed for gate failures so we don't double-post on drift. - Exclude retinascale-01-loaded: a fullPage HiDPI test whose captured height + DPR scaling vary run-to-run (~60% inter-run diff) — a flaky test, not render noise. IGNORE_SCREENSHOTS in config.ts; compare/promote/gen-manifest skip it; baseline removed. - Kill the baseline-dir shadowing: drop e2e/baseline-screenshots/ from BASELINE_DIRS + delete its 3 files (grid-tab-final, wxgrid-controls, wxgrid-dedicated-page) that duplicated names in baseline-screenshots/ with different bytes. - Prune 12 stale baselines (renamed/removed specs: wizard-01..04, gerbview-wizard-01..04, zoom-pl_editor-*, popup-03-palette). - Delete the dead compare-screenshots.sh / update-baseline-screenshots.sh. - Regenerate screenshot-manifest.json (355 entries). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Vaczi
parent
348596c515
commit
3efcdcf91c
24 changed files with 98 additions and 251 deletions
|
|
@ -1,108 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Compare test screenshots with baseline screenshots
|
||||
# This script compares all PNG files in baseline-screenshots with test-results
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
BASELINE_DIR="$PROJECT_ROOT/tests/baseline-screenshots"
|
||||
TEST_RESULTS_DIR="$PROJECT_ROOT/tests/test-results"
|
||||
|
||||
# Check if directories exist
|
||||
if [ ! -d "$BASELINE_DIR" ]; then
|
||||
echo "ERROR: Baseline directory not found: $BASELINE_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$TEST_RESULTS_DIR" ]; then
|
||||
echo "ERROR: Test results directory not found: $TEST_RESULTS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Screenshot Comparison ==="
|
||||
echo "Baseline: $BASELINE_DIR"
|
||||
echo "Test Results: $TEST_RESULTS_DIR"
|
||||
echo ""
|
||||
|
||||
# Counters
|
||||
total=0
|
||||
identical=0
|
||||
different=0
|
||||
missing_current=0
|
||||
extra_current=0
|
||||
|
||||
# Compare all baseline screenshots
|
||||
echo "=== Comparing Baseline Screenshots ==="
|
||||
for baseline in "$BASELINE_DIR"/*.png; do
|
||||
filename=$(basename "$baseline")
|
||||
current="$TEST_RESULTS_DIR/$filename"
|
||||
total=$((total + 1))
|
||||
|
||||
if [ ! -f "$current" ]; then
|
||||
echo "MISSING: $filename (not in test results)"
|
||||
missing_current=$((missing_current + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Compare using cmp (byte-by-byte comparison)
|
||||
if cmp -s "$baseline" "$current"; then
|
||||
identical=$((identical + 1))
|
||||
# Only show identical files if verbose
|
||||
if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then
|
||||
echo "IDENTICAL: $filename"
|
||||
fi
|
||||
else
|
||||
different=$((different + 1))
|
||||
|
||||
# Get file sizes
|
||||
baseline_size=$(stat -f%z "$baseline" 2>/dev/null || stat -c%s "$baseline")
|
||||
current_size=$(stat -f%z "$current" 2>/dev/null || stat -c%s "$current")
|
||||
diff_bytes=$((current_size - baseline_size))
|
||||
|
||||
# Calculate percent difference
|
||||
if [ "$baseline_size" -gt 0 ]; then
|
||||
diff_pct=$(echo "scale=2; ($diff_bytes * 100) / $baseline_size" | bc)
|
||||
else
|
||||
diff_pct="N/A"
|
||||
fi
|
||||
|
||||
echo "DIFFERENT: $filename (baseline: ${baseline_size}B, current: ${current_size}B, diff: ${diff_bytes}B / ${diff_pct}%)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Check for extra files in test results not in baseline
|
||||
echo ""
|
||||
echo "=== Checking for Extra Screenshots ==="
|
||||
for current in "$TEST_RESULTS_DIR"/*.png; do
|
||||
filename=$(basename "$current")
|
||||
baseline="$BASELINE_DIR/$filename"
|
||||
|
||||
if [ ! -f "$baseline" ]; then
|
||||
extra_current=$((extra_current + 1))
|
||||
current_size=$(stat -f%z "$current" 2>/dev/null || stat -c%s "$current")
|
||||
echo "EXTRA: $filename (${current_size}B, not in baseline)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
echo "Total baseline screenshots: $total"
|
||||
echo "Identical: $identical"
|
||||
echo "Different: $different"
|
||||
echo "Missing from test results: $missing_current"
|
||||
echo "Extra in test results: $extra_current"
|
||||
|
||||
# Exit with error if there are differences
|
||||
if [ "$different" -gt 0 ] || [ "$missing_current" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "WARNING: There are differences between baseline and test results!"
|
||||
exit 1
|
||||
else
|
||||
echo ""
|
||||
echo "SUCCESS: All screenshots match baseline!"
|
||||
exit 0
|
||||
fi
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Update baseline screenshots from test results
|
||||
# Only copies:
|
||||
# - NEW screenshots (not in baseline)
|
||||
# - Screenshots with SIGNIFICANT differences (>5% size change) when --all flag used
|
||||
#
|
||||
# Usage:
|
||||
# ./update-baseline-screenshots.sh # Only copy NEW screenshots
|
||||
# ./update-baseline-screenshots.sh --all # Copy new + significantly different
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TESTS_DIR="$SCRIPT_DIR/../tests"
|
||||
|
||||
COPY_ALL=0
|
||||
THRESHOLD=5 # Percent threshold for "significant" difference
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--all) COPY_ALL=1; shift ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SOURCE_DIR="$TESTS_DIR/test-results"
|
||||
DEST_DIR="$TESTS_DIR/baseline-screenshots"
|
||||
|
||||
if [ $COPY_ALL -eq 1 ]; then
|
||||
echo "Mode: Copy NEW + SIGNIFICANTLY DIFFERENT screenshots"
|
||||
else
|
||||
echo "Mode: Copy NEW screenshots only (use --all to include significant changes)"
|
||||
fi
|
||||
|
||||
if [ ! -d "$SOURCE_DIR" ]; then
|
||||
echo "Error: test-results directory not found at $SOURCE_DIR"
|
||||
echo "Run 'npm test' first to generate screenshots"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$DEST_DIR"
|
||||
|
||||
new_count=0
|
||||
updated_count=0
|
||||
|
||||
for src_file in "$SOURCE_DIR"/*.png; do
|
||||
[ -f "$src_file" ] || continue
|
||||
|
||||
filename=$(basename "$src_file")
|
||||
dest_file="$DEST_DIR/$filename"
|
||||
|
||||
if [ ! -f "$dest_file" ]; then
|
||||
# NEW: Screenshot doesn't exist in baseline
|
||||
echo "NEW: $filename"
|
||||
cp "$src_file" "$dest_file"
|
||||
((new_count++))
|
||||
elif [ $COPY_ALL -eq 1 ]; then
|
||||
# Check if significantly different
|
||||
src_size=$(stat -f%z "$src_file" 2>/dev/null || stat -c%s "$src_file")
|
||||
dest_size=$(stat -f%z "$dest_file" 2>/dev/null || stat -c%s "$dest_file")
|
||||
|
||||
if [ "$dest_size" -eq 0 ]; then
|
||||
diff_pct=100
|
||||
else
|
||||
diff=$((src_size - dest_size))
|
||||
diff=${diff#-} # Absolute value
|
||||
diff_pct=$((diff * 100 / dest_size))
|
||||
fi
|
||||
|
||||
if [ "$diff_pct" -ge "$THRESHOLD" ]; then
|
||||
echo "UPDATED ($diff_pct% diff): $filename"
|
||||
cp "$src_file" "$dest_file"
|
||||
((updated_count++))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
echo "New screenshots added: $new_count"
|
||||
if [ $COPY_ALL -eq 1 ]; then
|
||||
echo "Significantly changed: $updated_count"
|
||||
fi
|
||||
echo "Total in baseline: $(ls -1 "$DEST_DIR"/*.png 2>/dev/null | wc -l | tr -d ' ')"
|
||||
Loading…
Reference in a new issue