122 lines
4.6 KiB
Bash
Executable File
122 lines
4.6 KiB
Bash
Executable File
#!/bin/sh
|
|
# scripts/bite --- prove a fix's tests BITE: run them against the
|
|
# working tree (they must PASS), then against an older version of ONE
|
|
# file (they must FAIL).
|
|
#
|
|
# scripts/bite <ref> <path> [cargo-test-args...]
|
|
#
|
|
# Example (the PR #111 round-1 shape):
|
|
# scripts/bite HEAD~1 builtin/runtime/editops.lua \
|
|
# --test editops_acceptance -- capitalize trim_on_save_unexpected
|
|
#
|
|
# Exit status: 0 when the named tests pass now and fail against
|
|
# <ref>'s version of <path> (the fix bites), 1 when they still pass
|
|
# there (vacuous), 2 on usage/setup errors, 3 when the POSITIVE
|
|
# CONTROL fails. The working-tree file is restored on every exit path,
|
|
# including interrupts.
|
|
#
|
|
# THE POSITIVE CONTROL, and why it is not optional. Before this
|
|
# existed the script ran only the swapped tree, so it could not
|
|
# distinguish "my fix is load-bearing" from "my test is broken": a
|
|
# typo, an unrelated compile break, or a filter that matches nothing
|
|
# all made the swapped run fail, and a failing swapped run was the
|
|
# only thing it checked. It printed `bite: OK` and certified nothing.
|
|
# So the control asserts two things, because passing alone is not
|
|
# enough — `cargo test` with a filter matching zero tests exits 0, and
|
|
# then a compile error in the old tree would still read as OK:
|
|
#
|
|
# * the named tests PASS against the working tree, and
|
|
# * at least one test actually RAN.
|
|
#
|
|
# COMPILE errors of the old tree still count as "fails", and that is
|
|
# correct but weaker than a clean assertion failure --- the swapped
|
|
# file may simply not build against the current tree. The script now
|
|
# says which kind it saw instead of leaving it to be eyeballed. Prefer
|
|
# an assertion failure; treat a compile failure as a prompt to narrow
|
|
# the swap to a file that compiles both ways.
|
|
#
|
|
# Why this exists: bite-verification is step 4 of the working method,
|
|
# and the obvious shortcut --- git stash --- is a trap here. The stash
|
|
# namespace is REPO-GLOBAL: shared across every worktree and with
|
|
# humans, so a scripted push/pop can collide with (or pop!) someone
|
|
# else's stashed work. This helper never touches git state beyond a
|
|
# read-only `git show`, and it restores the working file from a
|
|
# mktemp copy --- NOT from git, so uncommitted work in <path> is
|
|
# preserved.
|
|
|
|
set -eu
|
|
|
|
if [ "$#" -lt 2 ]; then
|
|
echo "usage: scripts/bite <ref> <path> [cargo-test-args...]" >&2
|
|
exit 2
|
|
fi
|
|
|
|
ref=$1
|
|
path=$2
|
|
shift 2
|
|
|
|
if [ ! -f "$path" ]; then
|
|
echo "bite: no such file: $path" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# Sum the `N passed` figures across every `test result:` line, so a
|
|
# multi-target invocation is counted correctly rather than only its
|
|
# last binary.
|
|
count_passed() {
|
|
sed -n 's/^test result: ok\. \([0-9][0-9]*\) passed.*/\1/p' \
|
|
| awk '{ total += $1 } END { print total + 0 }'
|
|
}
|
|
|
|
# --- Positive control: the tests must pass, and must exist, NOW. ---
|
|
echo "bite: positive control --- running against the working tree" >&2
|
|
control_status=0
|
|
control_out=$(cargo test "$@" 2>&1) || control_status=$?
|
|
printf '%s\n' "$control_out"
|
|
|
|
if [ "$control_status" -ne 0 ]; then
|
|
echo "bite: NO CONTROL --- tests do not pass against the working tree." >&2
|
|
echo "bite: fix the tests first; a failure here makes the swapped run meaningless." >&2
|
|
exit 3
|
|
fi
|
|
|
|
control_ran=$(printf '%s\n' "$control_out" | count_passed)
|
|
if [ "$control_ran" -eq 0 ]; then
|
|
echo "bite: NO CONTROL --- the filter matched no tests (0 passed)." >&2
|
|
echo "bite: check the test names; an empty filter passes everywhere and proves nothing." >&2
|
|
exit 3
|
|
fi
|
|
echo "bite: control OK --- $control_ran test(s) pass against the working tree" >&2
|
|
|
|
saved=$(mktemp "${TMPDIR:-/tmp}/bite.XXXXXX")
|
|
cp -- "$path" "$saved"
|
|
restore() {
|
|
cp -- "$saved" "$path"
|
|
rm -f -- "$saved"
|
|
}
|
|
trap restore EXIT INT TERM
|
|
|
|
# The `./` prefix makes the pathspec cwd-relative for git-show, so the
|
|
# script works from any directory inside the repo.
|
|
git show "$ref:./$path" > "$path"
|
|
|
|
echo "bite: swapping in $ref:$path" >&2
|
|
swapped_status=0
|
|
swapped_out=$(cargo test "$@" 2>&1) || swapped_status=$?
|
|
printf '%s\n' "$swapped_out"
|
|
|
|
if [ "$swapped_status" -eq 0 ]; then
|
|
echo "bite: VACUOUS --- tests still pass against $ref:$path" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Distinguish the two ways the swapped run can fail. An assertion
|
|
# failure is the evidence we want; a compile failure only says the old
|
|
# file does not build here.
|
|
if printf '%s\n' "$swapped_out" | grep -q '^error\[E[0-9]*\]\|^error: could not compile'; then
|
|
echo "bite: OK (COMPILE) --- $ref:$path does not build against the current tree." >&2
|
|
echo "bite: weaker evidence than an assertion failure --- the tests may never have run." >&2
|
|
else
|
|
echo "bite: OK (assertion) --- tests fail against $ref:$path (the fix bites)"
|
|
fi
|