From ab87c99aafbce0a3086edc6d1c27d46f46f06fb7 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 21 Nov 2026 17:25:34 +0100 Subject: [PATCH] scripts: build rbgo every run instead of reusing a /tmp binary Five harnesses defaulted RBGO to a fixed /tmp path and reused whatever was already there: rubyspec/run.sh (/tmp/rbgo-rubyspec), rspec/run.sh, apps/run.sh and oracle.sh (/tmp/rbgo, which rspec/run.sh writes from any checkout), and bench/modules/run.sh (/tmp/rbgo, never built at all). On a machine with several checkouts, a sweep could measure another commit's interpreter and print the same line as a fresh build. With RBGO unset, each now builds ./cmd/rbgo from its own checkout into a mktemp -d directory and removes it on EXIT. An explicit RBGO is still honoured. rubyspec/run.sh and rspec/run.sh print the binary's vcs.revision/vcs.modified, read from the binary itself. rubyspec/run.sh also leaked files on every run. Its second trap replaced the first without naming the rbgo directory, and the final `exec` of the judge skipped EXIT traps altogether. The judge now runs as a child process followed by `exit $?`, and one trap removes RESULTS, the ratchet directory and the rbgo directory. The README's cmd/phantom recipe pointed -rbgo at /tmp/rbgo-rubyspec, which no longer exists after a run. It now builds its own binary. Closes #964 Co-Authored-By: Claude Opus 5.5 --- bench/modules/run.sh | 13 ++++++++- scripts/conformance/apps/run.sh | 16 +++++------ scripts/conformance/rspec/run.sh | 14 ++++++++-- scripts/conformance/rubyspec/README.md | 4 ++- scripts/conformance/rubyspec/run.sh | 38 ++++++++++++++++++++------ scripts/oracle.sh | 14 +++++----- 6 files changed, 70 insertions(+), 29 deletions(-) diff --git a/bench/modules/run.sh b/bench/modules/run.sh index 2d588629..0afc443b 100755 --- a/bench/modules/run.sh +++ b/bench/modules/run.sh @@ -31,12 +31,23 @@ # it falls hardest on the shortest-running one. set -u RUNS="${1:-5}" -RBGO="${RBGO:-/tmp/rbgo}" +RBGO="${RBGO:-}" RUBY="${RUBY:-ruby}" JRUBY="${JRUBY:-jruby}" TRUFFLE="${TRUFFLE:-truffleruby}" HERE="$(cd "$(dirname "$0")" && pwd)" +# #964: built EVERY run from this checkout, into a directory this run owns and +# deletes. It used to time whatever sat at /tmp/rbgo — built by any checkout, +# at any commit — and a benchmark of the wrong interpreter publishes as well as +# one of the right one. +if [ -z "$RBGO" ]; then + RBGO_DIR="$(mktemp -d)" + trap 'rm -rf "$RBGO_DIR"' EXIT + RBGO="$RBGO_DIR/rbgo" + (cd "$HERE/../.." && GOWORK=off go build -o "$RBGO" ./cmd/rbgo) || exit 1 +fi + # hires PROG ARGS... → "MILLISECONDS EXITSTATUS", with the child's own output # discarded inside the fork so that neither stream can reach this one. hires() { diff --git a/scripts/conformance/apps/run.sh b/scripts/conformance/apps/run.sh index cfb25245..b37a274a 100755 --- a/scripts/conformance/apps/run.sh +++ b/scripts/conformance/apps/run.sh @@ -19,7 +19,7 @@ # scripts/conformance/apps/run.sh --clone # clone missing libs (needs network) # scripts/conformance/apps/run.sh --only rake # one library # -# Environment overrides: RBGO (default /tmp/rbgo, else built from ./cmd/rbgo), +# Environment overrides: RBGO (default: built from ./cmd/rbgo every run), # MRI (default: ruby), CACHE (default /tmp/ger-apps-cache). set -eu @@ -41,14 +41,14 @@ while [ $# -gt 0 ]; do shift done +# #964: built EVERY run, into a directory this run owns and deletes. It used +# to prefer any /tmp/rbgo already present — another checkout's interpreter. if [ -z "$RBGO" ]; then - if [ -x /tmp/rbgo ]; then - RBGO=/tmp/rbgo - else - RBGO=/tmp/rbgo-apps - echo "building rbgo -> $RBGO" - (cd "$repo" && GOWORK=off go build -o "$RBGO" ./cmd/rbgo) - fi + RBGO_DIR=$(mktemp -d) + trap 'rm -rf "$RBGO_DIR"' EXIT + RBGO="$RBGO_DIR/rbgo" + echo "building rbgo -> $RBGO" + (cd "$repo" && GOWORK=off go build -o "$RBGO" ./cmd/rbgo) || exit 1 fi if ! command -v "$MRI" >/dev/null 2>&1; then diff --git a/scripts/conformance/rspec/run.sh b/scripts/conformance/rspec/run.sh index 9a02eaaa..4cf70556 100755 --- a/scripts/conformance/rspec/run.sh +++ b/scripts/conformance/rspec/run.sh @@ -11,24 +11,32 @@ # if MRI `ruby` is unavailable (rbgo-only run, no oracle diff). # # Env: -# RBGO path to rbgo binary (default: /tmp/rbgo, built if absent) +# RBGO path to rbgo binary (default: built from ./cmd/rbgo EVERY run) # RUBY path to MRI ruby (default: first `ruby` on PATH) # CLONES dir holding rspec-* repos (default: /tmp/rspec-clones) set -u HERE="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$HERE/../../.." && pwd)" -RBGO="${RBGO:-/tmp/rbgo}" +RBGO="${RBGO:-}" RUBY="${RUBY:-$(command -v ruby || true)}" CLONES="${CLONES:-/tmp/rspec-clones}" PARSESWEEP="${PARSESWEEP:-/tmp/parsesweep}" echo "== build rbgo ==" -if [ ! -x "$RBGO" ]; then +# #964: built EVERY run, into a directory this run owns and deletes. The old +# default reused any /tmp/rbgo already there, built by whichever checkout ran +# first; scripts/oracle.sh and apps/run.sh then picked that same file up. +if [ -z "$RBGO" ]; then + RBGO_DIR="$(mktemp -d)" + trap 'rm -rf "$RBGO_DIR"' EXIT + RBGO="$RBGO_DIR/rbgo" (cd "$REPO_ROOT" && GOWORK=off go build -o "$RBGO" ./cmd/rbgo) || exit 1 fi (cd "$REPO_ROOT" && GOWORK=off go build -o "$PARSESWEEP" ./scripts/conformance/rspec/parsesweep) || exit 1 echo "rbgo: $RBGO" +go version -m "$RBGO" | grep -E 'vcs\.(revision|modified)' || + echo " (no vcs stamp in $RBGO: built outside a git tree, or with -buildvcs=false)" echo "ruby: ${RUBY:-}" REPOS="rspec-support rspec-core rspec-expectations rspec-mocks" diff --git a/scripts/conformance/rubyspec/README.md b/scripts/conformance/rubyspec/README.md index 25860a8a..2af02118 100644 --- a/scripts/conformance/rubyspec/README.md +++ b/scripts/conformance/rubyspec/README.md @@ -238,8 +238,10 @@ a NoMethodError expectation it asks a real rbgo whether the method exists at all no instance is constructed), and reports the ones that do not. ```bash +# run.sh deletes the rbgo it built (#964), so build one from THIS checkout first +d="$(mktemp -d)" && go build -o "$d/rbgo" ./cmd/rbgo # the corpus run.sh pinned (macOS path; ~/.cache/rbgo/rubyspec-corpus on Linux) -go run ./scripts/conformance/rubyspec/cmd/phantom -root ~/Library/Caches/rbgo/rubyspec-corpus -rbgo /tmp/rbgo-rubyspec +go run ./scripts/conformance/rubyspec/cmd/phantom -root ~/Library/Caches/rbgo/rubyspec-corpus -rbgo "$d/rbgo" ``` Measured on `main` after #849: **59 attributable expectations over 2206 spec diff --git a/scripts/conformance/rubyspec/run.sh b/scripts/conformance/rubyspec/run.sh index 290ec844..7bd077a8 100755 --- a/scripts/conformance/rubyspec/run.sh +++ b/scripts/conformance/rubyspec/run.sh @@ -35,7 +35,7 @@ # for reproducibility. # # Env: -# RBGO path to the rbgo binary (default: built from ./cmd/rbgo) +# RBGO path to the rbgo binary (default: built from ./cmd/rbgo EVERY run) # SPECDIR ruby/spec checkout (default: CACHE; moved to SPEC_SHA) # CACHE clone cache dir (default: /rbgo/rubyspec-corpus) # JOBS parallelism (default: number of CPUs) @@ -45,7 +45,8 @@ set -u HERE="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$HERE/../../.." && pwd)" -RBGO="${RBGO:-/tmp/rbgo-rubyspec}" +RBGO="${RBGO:-}" +RBGO_DIR="" CACHE="${CACHE:-}" SPECDIR="${SPECDIR:-$CACHE}" TIMEOUT="${TIMEOUT:-25}" @@ -60,10 +61,21 @@ SPEC_URL="https://github.com/ruby/spec" BASELINE="$HERE/BASELINE" echo "== build rbgo ==" -if [ ! -x "$RBGO" ]; then +# #964: built EVERY run, into a directory this run owns and deletes. The old +# default reused whatever sat at /tmp/rbgo-rubyspec — the interpreter of the +# first checkout that ever ran this on the machine — and printed the same +# line either way. The build cache makes a rebuild cheap; a stale binary +# makes a whole sweep measure the wrong code. +if [ -z "$RBGO" ]; then + RBGO_DIR="$(mktemp -d)" + trap 'rm -rf "$RBGO_DIR"' EXIT # widened below once RESULTS exists + RBGO="$RBGO_DIR/rbgo" (cd "$REPO_ROOT" && GOWORK=off CGO_ENABLED=0 go build -o "$RBGO" ./cmd/rbgo) || exit 1 fi echo "rbgo: $RBGO" +# What was measured, read from the binary rather than from its path. +go version -m "$RBGO" | grep -E 'vcs\.(revision|modified)' || + echo " (no vcs stamp in $RBGO: built outside a git tree, or with -buildvcs=false)" echo echo "== ruby/spec corpus ==" @@ -79,7 +91,7 @@ SPECDIR="$(cd "$REPO_ROOT" && GOWORK=off go run ./scripts/conformance/rubyspec/c cp "$HERE/spec_helper.rb" "$SPECDIR/spec_helper.rb" RESULTS="$(mktemp)" -trap 'rm -f "$RESULTS"' EXIT +trap 'rm -f "$RESULTS"; if [ -n "$RBGO_DIR" ]; then rm -rf "$RBGO_DIR"; fi' EXIT run_one() { f="$1" @@ -114,8 +126,12 @@ find "$SPECDIR/language" "$SPECDIR/core" -name '*_spec.rb' | sort \ # Hand the sweep to the judge. Everything above this line runs processes; # everything below it decides, and that half is Go. -RATCHET="$(mktemp -d)/ratchet" -trap 'rm -f "$RESULTS" "$RATCHET"' EXIT +RATCHET_DIR="$(mktemp -d)" +RATCHET="$RATCHET_DIR/ratchet" +# One trap for everything this run created. It used to be replaced here by one +# that no longer named the rbgo directory, and the `exec` below skipped EXIT +# traps altogether, so every run left its binaries behind (#964). +trap 'rm -f "$RESULTS"; rm -rf "$RATCHET_DIR"; if [ -n "$RBGO_DIR" ]; then rm -rf "$RBGO_DIR"; fi' EXIT (cd "$REPO_ROOT" && GOWORK=off CGO_ENABLED=0 go build -o "$RATCHET" ./scripts/conformance/rubyspec/cmd/ratchet) || exit 2 echo @@ -124,8 +140,12 @@ if [ "${UPDATE_BASELINE:-0}" = "1" ]; then # refusal and the stamp both live in Go (cmd/ratchet, #876); this line only # carries the flag, so there is no second place where the rule could differ. if [ "${ABSORB:-0}" = "1" ]; then - exec "$RATCHET" -baseline "$BASELINE" -results "$RESULTS" -update -absorb + "$RATCHET" -baseline "$BASELINE" -results "$RESULTS" -update -absorb + exit $? fi - exec "$RATCHET" -baseline "$BASELINE" -results "$RESULTS" -update + "$RATCHET" -baseline "$BASELINE" -results "$RESULTS" -update + exit $? fi -exec "$RATCHET" -baseline "$BASELINE" -results "$RESULTS" +# Not exec: exec replaces the shell, and the EXIT trap above would never run. +"$RATCHET" -baseline "$BASELINE" -results "$RESULTS" +exit $? diff --git a/scripts/oracle.sh b/scripts/oracle.sh index 97e8365b..ec2cf1a0 100755 --- a/scripts/oracle.sh +++ b/scripts/oracle.sh @@ -12,7 +12,7 @@ # scripts/oracle.sh -b scripts/conformance/core_ext.txt # batch: one snippet # # per line (# = comment) # -# Environment overrides: RBGO (default: /tmp/rbgo, else built from ./cmd/rbgo), +# Environment overrides: RBGO (default: built from ./cmd/rbgo every run), # MRI (default: ruby), JRUBY (default: jruby), TRUFFLE (default: truffleruby). A # reference that is not installed is skipped with a note rather than failing the # run, so the oracle still works with whichever references are present. @@ -24,13 +24,13 @@ MRI=${MRI:-ruby} JRUBY=${JRUBY:-jruby} TRUFFLE=${TRUFFLE:-truffleruby} +# #964: built EVERY run, into a directory this run owns and deletes. It used +# to prefer any /tmp/rbgo already present — another checkout's interpreter. if [ -z "$RBGO" ]; then - if [ -x /tmp/rbgo ]; then - RBGO=/tmp/rbgo - else - RBGO=/tmp/rbgo-oracle - (cd "$here" && GOWORK=off go build -o "$RBGO" ./cmd/rbgo) - fi + RBGO_DIR=$(mktemp -d) + trap 'rm -rf "$RBGO_DIR"' EXIT + RBGO="$RBGO_DIR/rbgo" + (cd "$here" && GOWORK=off go build -o "$RBGO" ./cmd/rbgo) || exit 1 fi usage() {