From a129fdc3d10f70dc47ff8b9df797858ddf0f5e45 Mon Sep 17 00:00:00 2001 From: Marcus Kainth Date: Fri, 25 Sep 2026 14:30:39 +0100 Subject: [PATCH] ci: record each main commit's parity and cost nightly The nightly gains two jobs. native-regression (contents: read) walks the main commits not yet recorded, oldest first and first parents only, at most eight a night (max_commits), with scripts/native-regression-walk.sh. Each commit is built in its own worktree, loaded into a fresh database and run with `native diff --record` against a probe trace generated from that commit's ROM and probe; traces are cached by the git hashes of rom/PINNED_HASH and refemu/probe/. `clickdoom native regress` then judges the night against the recorded lines. native diff compares nothing when its span holds a refusal, so the walk diffs twice: SEARCH_TICS tics (2000) to find the first refused tic T, then T-1 tics, and keeps the second line with the first one's refusal. It refuses a second line that reaches T or compared no tic. A first line that did compare is kept as it is. The walk measures the last recorded commit again before the new ones, so the first new commit has a parent measured in the same job on the same VM; regress judges cost only against such a parent. A commit that fails to build, load or run becomes an error line and the walk goes on. The walk stops starting commits after BUDGET_MINUTES (240), so a slow night still uploads what it did. native-record (contents: write) downloads the artifact and appends the lines whose commit is not yet recorded to native/bench/regression/results.jsonl on the regression-data branch, creating it as an orphan branch through the git data API the first time. It checks nothing out and runs no repository code. The branch is never merged into main. nightly.yml's header, CONTRIBUTING.md and DEVELOPING.md describe the new jobs in the same change. --- .github/workflows/nightly.yml | 173 ++++++++++++++++++++++++++-- CONTRIBUTING.md | 5 +- DEVELOPING.md | 28 ++++- scripts/native-regression-walk.sh | 182 ++++++++++++++++++++++++++++++ 4 files changed, 377 insertions(+), 11 deletions(-) create mode 100755 scripts/native-regression-walk.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 56eebdc2..a9de7905 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,14 +1,18 @@ -# The long differential run, nightly. It does not block merges. +# The long runs, nightly. They do not block merges. # -# This is the only job that compares memory. ci.yml's differential-smoke stops -# well short of the first RAM_HASH_INTERVAL boundary, so `ramhash` and `fbhash` -# are exercised here and nowhere else. A failure here needs a divergence-report -# issue opened by hand; nothing files one automatically. +# deep-diff is the only job that compares memory. ci.yml's differential-smoke +# stops well short of the first RAM_HASH_INTERVAL boundary, so `ramhash` and +# `fbhash` are exercised here and nowhere else. A failure there needs a +# divergence-report issue opened by hand. # -# Throughput is deliberately not benchmarked here. A shared runner cannot give -# a timing a quiet machine gives, and a gate that fails for reasons unrelated to -# the change is worse than no gate. Run `bench-canonical-throughput` on a quiet -# box instead. +# native-regression runs `native diff --record` on each main commit not yet +# recorded and judges each line against the one before it. native-record +# appends the lines to the `regression-data` branch. Cost is judged only +# between a parent and its child measured one after the other in the same +# job, never against a number from another runner. +# +# Throughput is not benchmarked here. A shared runner cannot give the timing a +# quiet machine gives. Run `bench-canonical-throughput` on a quiet box instead. name: nightly @@ -25,6 +29,9 @@ on: # 91.6 minutes, a 3.28x margin inside the 300-minute timeout, and # crosses 4 RAM_HASH_INTERVAL boundaries. default: "5000000" + max_commits: + description: "Main commits native-regression walks after the last recorded one" + default: "8" concurrency: group: nightly-${{ github.ref }} @@ -37,6 +44,10 @@ env: # Local-only. This database holds an emulator's RAM and nothing secret. CLICKHOUSE_PASSWORD: clickdoom CLICKHOUSE_DATABASE: clickdoom + # The branch the native-regression lines are kept on, and the file on it. + # The branch is never merged into main. + REGRESSION_BRANCH: regression-data + REGRESSION_PATH: native/bench/regression/results.jsonl jobs: deep-diff: @@ -65,3 +76,147 @@ jobs: ./target/release/clickdoom emulation diff "$INSTRUCTIONS" --host localhost --port 8123 \ --bin rom/build/doom-rv32im.bin --manifest rom/build/manifest.json \ --refemu-bin target/release/refemu + + native-regression: + # Reads the repository and writes nothing to it. What it found goes to + # native-record as an artifact. + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 330 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - name: Start the pinned ClickHouse + run: make up + - name: Build ROM + run: make -C rom + - name: Build the judge + # The walk rebuilds target/release/clickdoom for every commit, so the + # binary that judges the night is kept apart. + run: | + cargo build --locked --release -p clickdoom-driver + cp target/release/clickdoom "$RUNNER_TEMP/clickdoom-judge" + - name: Name main's probe trace + id: probe + run: | + echo "key=$(git rev-parse origin/main:rom/PINNED_HASH)-$(git rev-parse origin/main:refemu/probe)" >>"$GITHUB_OUTPUT" + - name: Restore the probe traces + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/traces + key: probe-trace-${{ steps.probe.outputs.key }} + - name: Read what is recorded + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + if gh api "repos/$REPO/branches/$REGRESSION_BRANCH" --silent 2>"$RUNNER_TEMP/err"; then + gh api -H "Accept: application/vnd.github.raw" \ + "repos/$REPO/contents/$REGRESSION_PATH?ref=$REGRESSION_BRANCH" >"$RUNNER_TEMP/history.jsonl" + echo "$(wc -l <"$RUNNER_TEMP/history.jsonl") line(s) recorded" + elif grep -q "HTTP 404" "$RUNNER_TEMP/err"; then + echo "no $REGRESSION_BRANCH branch yet" + : >"$RUNNER_TEMP/history.jsonl" + else + cat "$RUNNER_TEMP/err" >&2 + exit 1 + fi + - name: Walk the commits not yet recorded + env: + MAX_COMMITS: ${{ github.event.inputs.max_commits || '8' }} + TRACES: ${{ runner.temp }}/traces + CLICKDOOM_DATABASE: regress + run: scripts/native-regression-walk.sh "$RUNNER_TEMP/history.jsonl" regression + - name: Judge the night + run: | + if [ ! -s regression/night.jsonl ]; then + echo "nothing walked, so nothing to judge" + exit 0 + fi + history=() + if [ -s "$RUNNER_TEMP/history.jsonl" ]; then + history=(--history "$RUNNER_TEMP/history.jsonl") + fi + status=0 + "$RUNNER_TEMP/clickdoom-judge" native regress regression/night.jsonl "${history[@]}" \ + || status=$? + case "$status" in + 0) ;; + 3) echo "::warning::a recorded line regressed; the lines above name it" ;; + *) exit "$status" ;; + esac + - name: Upload what the night found + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-regression + path: regression/ + if-no-files-found: error + + native-record: + # Writes the branch, and runs no repository code: it checks nothing out, + # and reads only the artifact native-regression uploaded. + needs: native-regression + runs-on: ubuntu-latest + permissions: + contents: write # appends to the regression-data branch + timeout-minutes: 10 + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + steps: + - name: Download what the night found + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: native-regression + path: regression + - name: Append the new lines to the data branch + run: | + if [ ! -s regression/night.jsonl ]; then + echo "nothing walked, so nothing to record" + exit 0 + fi + file_sha="" + if gh api "repos/$REPO/branches/$REGRESSION_BRANCH" --silent 2>err; then + gh api -H "Accept: application/vnd.github.raw" \ + "repos/$REPO/contents/$REGRESSION_PATH?ref=$REGRESSION_BRANCH" >history.jsonl + file_sha=$(gh api "repos/$REPO/contents/$REGRESSION_PATH?ref=$REGRESSION_BRANCH" --jq .sha) + elif grep -q "HTTP 404" err; then + : >history.jsonl + else + cat err >&2 + exit 1 + fi + # A commit the branch already holds is not written twice. The walk + # measures the last recorded commit again as the next one's parent. + jq -c -n --slurpfile old history.jsonl --slurpfile new regression/night.jsonl ' + reduce $new[] as $line ({seen: ($old | map(.commit)), out: []}; + if (.seen | index($line.commit)) then . else + .seen += [$line.commit] | .out += [$line] end) | .out[]' >fresh.jsonl + count=$(wc -l results.jsonl + message="regression: record $count commit(s) from run $RUN_ID" + if [ -n "$file_sha" ]; then + gh api -X PUT "repos/$REPO/contents/$REGRESSION_PATH" --silent \ + -f message="$message" -f branch="$REGRESSION_BRANCH" -f sha="$file_sha" \ + -f content="$(base64 -w0 results.jsonl)" + else + # The branch is an orphan: its one file and no history from main. + blob=$(gh api "repos/$REPO/git/blobs" -f encoding=base64 \ + -f content="$(base64 -w0 results.jsonl)" --jq .sha) + tree=$(gh api "repos/$REPO/git/trees" -f "tree[][path]=$REGRESSION_PATH" \ + -f "tree[][mode]=100644" -f "tree[][type]=blob" -f "tree[][sha]=$blob" --jq .sha) + commit=$(gh api "repos/$REPO/git/commits" -f message="$message" -f tree="$tree" --jq .sha) + gh api "repos/$REPO/git/refs" --silent \ + -f ref="refs/heads/$REGRESSION_BRANCH" -f sha="$commit" + fi + echo "recorded $count line(s) on $REGRESSION_BRANCH" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4334c708..8661b34f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,10 @@ what runs there. Expect upwards of ten minutes, most of it the ROM build and the differential smoke. It is necessary and not sufficient. The nightly deep-diff is the only run that -compares memory, and no pull request makes it. +compares memory, and no pull request makes it. The nightly also diffs native +mode against the reference emulator on each new main commit, and names the +commit whose first refusal or first divergence moved earlier, or whose tic +statement got slower than its parent's. Check by exit code. A pipeline reports only its last command's status, so `make gates | tail` can hide a failure. diff --git a/DEVELOPING.md b/DEVELOPING.md index 565c4f67..7087e62f 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -31,7 +31,7 @@ ROM build and the smoke diff. `lint` also runs `check-adr`, `actionlint` and `check-bare`, which have no CI job of their own. -The benches, `fuzz`, the milestone targets and the nightly deep-diff sit outside +The benches, `fuzz`, the milestone targets and the nightly jobs sit outside `gates`, by cost or by what they need. A timing run needs a quiet machine, and the deep-diff takes hours. @@ -177,6 +177,32 @@ The melt's pass count per frame comes from the reference run and is loaded as data from `driver/melt/demo3.tsv`; its provenance is in that directory's README. +### What the nightly records + +`nightly.yml`'s `native-regression` job runs `scripts/native-regression-walk.sh` +over the main commits not yet recorded, oldest first, at most `max_commits` a +night (default 8). Each commit is built in its own worktree and run with +`native diff --record` against a probe trace generated from that commit's ROM +and probe. A diff that stops at a refusal compares nothing, so the walk runs a +second diff over the tics before the refused one and keeps its line with the +first diff's refusal. `driver/src/native/record.rs` documents the line. A commit +that does not build or run gets a line with `error`, and the walk goes on. + +`clickdoom native regress` judges the night's lines against the recorded ones, +and its `--help` states the rules. Cost is judged only between a parent and a +child measured one after the other in the same job, so the walk measures the +last recorded commit again first. + +The `native-record` job appends the new lines to +`native/bench/regression/results.jsonl` on the `regression-data` branch, which +is never merged into main. It is the only job with write access, and it runs no +repository code. With `max_commits` at 0 the walk measures only the last +recorded commit. + +The walk runs locally against `make up`: +`scripts/native-regression-walk.sh HISTORY OUT`, with `HISTORY` a copy of the +branch's file. + ## Benchmarks Timings need a quiet machine, and the numbers in `docs/experiments/` were diff --git a/scripts/native-regression-walk.sh b/scripts/native-regression-walk.sh new file mode 100755 index 00000000..2ae9b5a7 --- /dev/null +++ b/scripts/native-regression-walk.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# +# Records native mode's parity and cost for each main commit not yet +# recorded, oldest first: one `clickdoom native diff --record` line per +# commit. +# +# scripts/native-regression-walk.sh HISTORY OUT +# +# HISTORY holds the lines recorded so far and may be absent. The walk starts +# at the last commit it records, measured again so the commit after it has a +# parent measured on the same machine, and follows TIP's first parents from +# there. With no HISTORY, or when its last commit is not an ancestor of TIP, +# it measures TIP's first parent and TIP. +# +# Each commit is checked out in its own worktree, built, loaded into a fresh +# database and diffed against a probe trace generated from that commit's ROM +# and probe. The first diff runs SEARCH_TICS tics to find the first refused +# tic. When its line compared nothing, a second diff runs over the tics +# before the refusal, and the line kept is the second one carrying the +# first one's refusal. That line has to end below the refusal and compare at +# least one tic. +# +# A commit that does not build, load or run gets a line with `error` instead, +# and the walk goes on. OUT/night.jsonl gets one line per commit walked, and +# OUT/logs/ each commit's output. +# +# Environment: +# TIP the commit to walk up to, default origin/main +# MAX_COMMITS commits to walk after the parent, default 8 +# BUDGET_MINUTES no commit starts once the walk has run this long, default 240 +# SEARCH_TICS the first diff's span, default 2000 +# TRACES where probe traces are kept by ROM and probe version, +# default OUT/traces +# CLICKDOOM_DATABASE the database each commit loads, default regress +# CH_HOST, CH_HTTP_PORT and CLICKHOUSE_PASSWORD, as for make +set -euo pipefail +cd "$(dirname "$0")/.." + +if [ "$#" -ne 2 ]; then + echo "usage: $0 HISTORY OUT" >&2 + exit 2 +fi +history=$1 +mkdir -p "$2" +out=$(cd "$2" && pwd) +max_commits=${MAX_COMMITS:-8} +budget_minutes=${BUDGET_MINUTES:-240} +search_tics=${SEARCH_TICS:-2000} +traces=${TRACES:-$out/traces} +tip=${TIP:-origin/main} +conn=(--host "${CH_HOST:-localhost}" --port "${CH_HTTP_PORT:-8123}" + --database "${CLICKDOOM_DATABASE:-regress}") +export CARGO_TARGET_DIR="$PWD/target" + +mkdir -p "$out/logs" "$traces" +night="$out/night.jsonl" +: >"$night" +work=$(mktemp -d) +trap 'rm -rf "$work"; git worktree prune' EXIT + +last="" +if [ -s "$history" ]; then + last=$(tail -n 1 "$history" | jq -r .commit) +fi +if [ -n "$last" ] && git merge-base --is-ancestor "$last" "$tip" 2>/dev/null; then + parent=$last + mapfile -t newer < <(git rev-list --first-parent --reverse "$last..$tip") +else + parent=$(git rev-parse "$tip^") + newer=("$(git rev-parse "$tip")") +fi +if [ "${#newer[@]}" -eq 0 ]; then + echo "nothing on $tip after $last" + exit 0 +fi +if [ "${#newer[@]}" -gt "$max_commits" ]; then + echo "${#newer[@]} commits after $parent; walking the oldest $max_commits" + newer=("${newer[@]:0:$max_commits}") +fi + +# The ROM this job built, which a commit pinning the same hash reuses. +built_rom="" +if [ -f rom/build/doom-rv32im.bin ]; then + built_rom=$(sha256sum rom/build/doom-rv32im.bin | cut -d' ' -f1) +fi + +# Records why the step that called it failed, for the error line. +fail() { + printf '%s' "$1" >"$work/why" + return 1 +} + +# One diff of SPAN tics in WT, writing its line to RECORD. Exit 3 is a +# refusal or a divergence, which the line records. Any other exit is a +# failure. +diff_span() { + local wt=$1 bin=$2 trace=$3 span=$4 record=$5 status=0 + (cd "$wt" && "$bin/clickdoom" native diff "$span" --probe "$trace" \ + "${conn[@]}" --record "$record") || status=$? + if [ "$status" -ne 0 ] && [ "$status" -ne 3 ]; then + fail "native diff $span exited $status" + elif [ ! -f "$record" ] || [ "$(wc -l <"$record")" -ne 1 ]; then + fail "native diff $span wrote no line" + fi +} + +# Builds and measures SHA, and writes the line to keep to $work/SHA.line. +measure() { + local sha=$1 wt="$work/$1" bin="$work/bin/$1" + git worktree add --detach --quiet "$wt" "$sha" || fail "checkout failed" || return + (cd "$wt" && cargo build --locked --release -p clickdoom-driver -p refemu) \ + || fail "cargo build failed" || return + mkdir -p "$bin" + cp "$CARGO_TARGET_DIR/release/clickdoom" "$CARGO_TARGET_DIR/release/refemu" "$bin/" + + if [ "$(cat "$wt/rom/PINNED_HASH")" = "$built_rom" ]; then + mkdir -p "$wt/rom/build" + cp rom/build/doom-rv32im.bin rom/build/doom-rv32im.elf rom/build/manifest.json "$wt/rom/build/" + else + make -C "$wt/rom" || fail "the ROM build failed" || return + fi + + # A trace depends only on the ROM and on the probe that dumps it. + local rom_blob probe_tree trace + rom_blob=$(git rev-parse "$sha:rom/PINNED_HASH") || fail "no rom/PINNED_HASH" || return + probe_tree=$(git rev-parse "$sha:refemu/probe") || fail "no refemu/probe" || return + trace="$traces/$rom_blob-$probe_tree.tsv" + if [ ! -f "$trace" ]; then + make -C "$wt" gen-probe-trace REFEMU="$bin/refemu" \ + || fail "make gen-probe-trace failed" || return + cp "$wt/refemu/reference_traces/demo3/probe.$(cut -c1-12 "$wt/rom/PINNED_HASH").tsv" "$trace" \ + || fail "make gen-probe-trace wrote no trace" || return + fi + + (cd "$wt" && "$bin/clickdoom" native load --fresh "${conn[@]}") \ + || fail "native load failed" || return + + local search="$work/$sha.search.jsonl" span="$work/$sha.span.jsonl" + local line="$work/$sha.line" refused + diff_span "$wt" "$bin" "$trace" "$search_tics" "$search" || return + if jq -e '.compared_through != null' "$search" >/dev/null; then + cp "$search" "$line" + else + refused=$(jq -r '.first_refused_tic' "$search") + if [ "$refused" -le 1 ]; then + fail "tic $refused refused, so no tic can be compared" + return + fi + diff_span "$wt" "$bin" "$trace" "$((refused - 1))" "$span" || return + jq -e --argjson refused "$refused" \ + '.first_refused_tic == null and .compared_through < $refused and .compared_tics > 0' \ + "$span" >/dev/null \ + || fail "native diff $((refused - 1)) did not compare the tics before the refusal at $refused" \ + || return + jq -c -s '.[1] + {first_refused_tic: .[0].first_refused_tic, first_refused_bits: .[0].first_refused_bits}' \ + "$search" "$span" >"$line" + fi + [ "$(jq -r .commit "$line")" = "$sha" ] \ + || fail "the line names commit $(jq -r .commit "$line"), not $sha" +} + +for sha in "$parent" "${newer[@]}"; do + if [ "$SECONDS" -ge "$((budget_minutes * 60))" ]; then + echo "stopping after $budget_minutes minutes; the next run starts from the last line written" + break + fi + short=${sha:0:12} + echo "::group::$short" + rm -f "$work/why" + if measure "$sha" 2>&1 | tee "$out/logs/$short.log"; then + cat "$work/$sha.line" >>"$night" + echo "$short: $(cat "$work/$sha.line")" + else + why=$(cat "$work/why" 2>/dev/null || echo "failed; see logs/$short.log") + echo "::error::$short: $why" + jq -n -c --arg commit "$sha" --arg error "$why" --arg run_id "${GITHUB_RUN_ID:-}" \ + '{commit: $commit, error: $error, run_id: (if $run_id == "" then null else $run_id end)}' >>"$night" + fi + git worktree remove --force "$work/$sha" 2>/dev/null || true + echo "::endgroup::" +done +echo "$(wc -l <"$night") line(s) in $night"