From 535e4a9f9e4cd7cba9606172796fb640193f1c09 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Fri, 12 Jun 2026 18:02:35 +0100 Subject: [PATCH 1/4] feat: only check crates that have changed in their dependency structure Signed-off-by: Giles Cope --- .../continuous-integration-checks.yml | 45 ------ .github/workflows/feature-unification.yml | 81 ++++++++++ Earthfile | 12 +- scripts/feature-unification-scope.sh | 144 ++++++++++++++++++ 4 files changed, 235 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/feature-unification.yml create mode 100755 scripts/feature-unification-scope.sh diff --git a/.github/workflows/continuous-integration-checks.yml b/.github/workflows/continuous-integration-checks.yml index cbfa5a895..be482d9d0 100644 --- a/.github/workflows/continuous-integration-checks.yml +++ b/.github/workflows/continuous-integration-checks.yml @@ -63,48 +63,3 @@ jobs: if: steps.guard.outputs.hit != 'true' with: key: ${{ steps.guard.outputs.key }} - - feature-unification: - name: Feature Unification Check - runs-on: ubuntu-latest - permissions: - contents: read - env: - FORCE_COLOR: 1 - steps: - - name: Checkout node repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3 - with: - submodules: true - - - id: guard - uses: ./.github/actions/tree-cache-guard - - - name: Login to GHCR - if: steps.guard.outputs.hit != 'true' - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee #v4.2.0 - with: - registry: ghcr.io - username: MidnightCI - password: ${{ secrets.MIDNIGHTCI_PACKAGES_READ }} - - - uses: EarthBuild/actions-setup@cae2d9ab68894d8402751fe42e07c7cca0272f7f - if: steps.guard.outputs.hit != 'true' - with: - version: v0.8.16 - github-token: ${{ github.token }} - use-cache: false - - - name: Free disk space - if: steps.guard.outputs.hit != 'true' - run: scripts/free-disk-space.sh - - - name: Run feature unification check - if: steps.guard.outputs.hit != 'true' - run: | - . ./.envrc && earthly --ci +check-feature-unification - - - uses: ./.github/actions/tree-cache-guard/save - if: steps.guard.outputs.hit != 'true' - with: - key: ${{ steps.guard.outputs.key }} diff --git a/.github/workflows/feature-unification.yml b/.github/workflows/feature-unification.yml new file mode 100644 index 000000000..11d15d81e --- /dev/null +++ b/.github/workflows/feature-unification.yml @@ -0,0 +1,81 @@ +name: Feature Unification + +# Advisory check (not in the main ruleset's required status checks): verifies +# each crate compiles without dev-deps via `cargo hack check --no-dev-deps`. +# Feature-unification regressions are almost always introduced by manifest +# changes, so this only runs on PRs that touch one, scoped to the +# reverse-dependency closure of the changed crates. Deliberately not run on +# main pushes or merge queues. + +on: + pull_request: + branches: ["**"] + paths: + - "**/Cargo.toml" + - "Cargo.lock" + - "Earthfile" + - ".github/workflows/feature-unification.yml" + +# no top level default permissions for security reasons +permissions: {} +concurrency: + group: ${{ format('{0}-{1}', github.workflow, github.head_ref) }} + cancel-in-progress: true +jobs: + feature-unification: + name: Feature Unification Check + runs-on: ubuntu-latest + permissions: + contents: read + env: + FORCE_COLOR: 1 + steps: + - name: Checkout node repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3 + with: + submodules: true + # The merge commit plus both parents, so HEAD^1 (the base) is + # available for the scope diff below. + fetch-depth: 2 + + - id: guard + uses: ./.github/actions/tree-cache-guard + + # Only check the reverse-dependency closure of the crates the PR diff + # touches (see scripts/feature-unification-scope.sh). + - id: scope + if: steps.guard.outputs.hit != 'true' + env: + # Skip the rust-toolchain.toml pin: the runner's preinstalled + # stable is plenty for `cargo metadata` and saves a 1.95 download. + RUSTUP_TOOLCHAIN: stable + run: | + packages=$(git diff --name-only HEAD^1 HEAD | ./scripts/feature-unification-scope.sh) + echo "scope: ${packages:-}" + echo "packages=$packages" >> "$GITHUB_OUTPUT" + + # No GHCR login: the build chain only pulls the public + # midnightntwrk/midnight-node-ci image from docker.io. + - uses: EarthBuild/actions-setup@cae2d9ab68894d8402751fe42e07c7cca0272f7f + if: steps.guard.outputs.hit != 'true' && steps.scope.outputs.packages != '' + with: + version: v0.8.16 + github-token: ${{ github.token }} + use-cache: false + + # Uncomment if disk space again becomes an issue. + # - name: Free disk space + # if: steps.guard.outputs.hit != 'true' && steps.scope.outputs.packages != '' + # run: scripts/free-disk-space.sh + + - name: Run feature unification check + if: steps.guard.outputs.hit != 'true' && steps.scope.outputs.packages != '' + env: + PACKAGES: ${{ steps.scope.outputs.packages }} + run: | + . ./.envrc && earthly --ci +check-feature-unification --PACKAGES="$PACKAGES" + + - uses: ./.github/actions/tree-cache-guard/save + if: steps.guard.outputs.hit != 'true' + with: + key: ${{ steps.guard.outputs.key }} diff --git a/Earthfile b/Earthfile index 92ef2007f..950ad0c99 100644 --- a/Earthfile +++ b/Earthfile @@ -856,6 +856,8 @@ check-rust: # check-feature-unification verifies each crate compiles without dev-deps, # catching issues where workspace feature unification masks missing dependencies. +# The partner-chains demo crates are excluded: they are upstream examples, not +# shipped artifacts, and cost ~5min of the serial check. check-feature-unification: FROM +check-rust-prepare CACHE --sharing shared --id cargo-git /usr/local/cargo/git @@ -867,8 +869,14 @@ check-feature-unification: ENV SKIP_WASM_BUILD=1 ENV CARGO_INCREMENTAL=0 - RUN cargo binstall --no-confirm cargo-hack - RUN cargo hack check --workspace --no-dev-deps + # Package scope: full workspace by default; PR builds pass the output of + # scripts/feature-unification-scope.sh (reverse-dependency closure of the + # crates touched by the diff) to skip re-checking unaffected crates. + ARG PACKAGES="--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" + # Pinned: an unpinned binstall here can drift from the version baked into + # the CI base image and change check behaviour between runs. + RUN cargo binstall --no-confirm --locked cargo-hack@0.6.45 + RUN cargo hack check $PACKAGES --no-dev-deps # check-metadata confirms that metadata in the repo matches a given node image check-metadata: diff --git a/scripts/feature-unification-scope.sh b/scripts/feature-unification-scope.sh new file mode 100755 index 000000000..e8831da03 --- /dev/null +++ b/scripts/feature-unification-scope.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Compute the cargo-hack package scope for the feature-unification check. +# +# Reads changed file paths (one per line) on stdin and prints the package +# selection args for `cargo hack check --no-dev-deps`: +# +# * "--workspace --exclude ..." a global build input changed, check everything +# * "-p a -p b ..." only these crates and their reverse-dependency +# closure need re-checking +# * "" (empty) nothing compile-relevant changed, skip the check +# +# Why the reverse closure is sufficient: the check resolves features +# per-package, so a change in crate X can only alter the outcome for X itself +# or for crates whose dependency graph contains X. Dev-dependency edges are +# ignored on purpose -- the check strips dev-deps, so X can never enter a +# dependent's checked graph through one. +# +# Files inside a crate directory attribute to that crate (even data/markdown: +# they may be include_str!'d). Files that belong to no crate and match no +# IGNORE pattern force a full --workspace run. +# +# Self-test: scripts/feature-unification-scope.sh --self-test +set -euo pipefail + +# jq program. stdin: `cargo metadata --no-deps` JSON. $changed: array of paths. +# shellcheck disable=SC2016 # single quotes are deliberate: this is jq, not shell +JQ_PROG=' +"--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" + as $workspace_args +| ["partner-chains-demo-node", "partner-chains-demo-runtime"] as $excluded +# A change to any of these invalidates every crate'\''s resolution. +| ["^Cargo\\.toml$", "^Cargo\\.lock$", "^\\.cargo/", "^\\.config/", + "^rust-toolchain", "^Earthfile$", "^scripts/feature-unification-scope\\.sh$", + "^\\.github/workflows/feature-unification\\.yml$"] as $global +# Never compile-relevant (only consulted for files outside every crate dir). +| ["^changes/", "^\\.changes_archive/", "^\\.github/", "\\.md$", "^LICENSE"] as $ignore +| .workspace_root as $root +| [.packages[] | {name, dir: (.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr($root + "/"))}] + as $pkgs +| ($pkgs | map(.name)) as $names +# name -> [workspace deps], normal/build kinds only (see header for why not dev) +| (reduce .packages[] as $p ({}; + .[$p.name] = [$p.dependencies[] + | select((.kind == null or .kind == "build") and (.name as $n | $names | index($n))) + | .name])) + as $deps +| if any($changed[]; . as $f | any($global[]; . as $re | $f | test($re))) then + $workspace_args + else + # Attribute each file to the crate with the longest matching dir prefix, + # so nested crates (e.g. pallets/x/mock) win over their parent. + ([$changed[] | . as $f + | ([$pkgs[] | select((.dir + "/") as $pre | $f | startswith($pre))] | max_by(.dir | length)) + | if . != null then {pkg: .name} + elif any($ignore[]; . as $re | $f | test($re)) then empty + else {unattributable: $f} + end]) + as $hits + | if any($hits[]; .unattributable) then $workspace_args + else + ([$hits[].pkg] | unique) as $seed + # Reverse-dependency closure: grow until no new dependent is added. + | {cur: $seed, grew: ($seed | length > 0)} + | until(.grew | not; + . as $s + | ([$names[] | select(. as $n + | ($s.cur | index($n) | not) + and ($deps[$n] | any(. as $d | $s.cur | index($d))))]) + as $more + | {cur: ((.cur + $more) | sort), grew: ($more | length > 0)}) + | (.cur - $excluded) as $closure + | if $closure == [] then "" + elif $closure == (($names - $excluded) | sort) then $workspace_args + else [$closure[] | "-p " + .] | join(" ") + end + end + end +' + +scope() { # $1: changed paths as JSON array; metadata JSON on stdin + jq -r --argjson changed "$1" "$JQ_PROG" +} + +self_test() { + local meta ws got n=0 + ws="--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" + # base <- mid <- leaf; dev-user dev-depends on base; loner is isolated; + # demo crates are the excluded ones. + meta='{ + "workspace_root": "/ws", + "packages": [ + {"name": "base", "manifest_path": "/ws/base/Cargo.toml", "dependencies": []}, + {"name": "mid", "manifest_path": "/ws/mid/Cargo.toml", + "dependencies": [{"name": "base", "kind": null}]}, + {"name": "leaf", "manifest_path": "/ws/leaf/Cargo.toml", + "dependencies": [{"name": "mid", "kind": null}]}, + {"name": "loner", "manifest_path": "/ws/loner/Cargo.toml", "dependencies": []}, + {"name": "dev-user", "manifest_path": "/ws/dev-user/Cargo.toml", + "dependencies": [{"name": "base", "kind": "dev"}]}, + {"name": "partner-chains-demo-node", "manifest_path": "/ws/demo/node/Cargo.toml", + "dependencies": [{"name": "base", "kind": null}]}, + {"name": "partner-chains-demo-runtime", "manifest_path": "/ws/demo/runtime/Cargo.toml", + "dependencies": []} + ] + }' + run_case() { # $1: changed files (space-sep), $2: expected output + local files_json + # shellcheck disable=SC2086 # splitting $1 into one path per line is the point + files_json=$(printf '%s\n' $1 | jq -R -s 'split("\n") | map(select(length > 0))') + got=$(echo "$meta" | scope "$files_json") + if [[ "$got" != "$2" ]]; then + echo "self-test FAIL for [$1]: expected '$2', got '$got'" >&2 + exit 1 + fi + n=$((n + 1)) + } + # change in base ripples up through mid to leaf; not to the dev-only + # user, the loner, or the excluded demo crate + run_case "base/src/lib.rs" "-p base -p leaf -p mid" + run_case "leaf/src/lib.rs" "-p leaf" + # crate manifest change scopes like a source change + run_case "mid/Cargo.toml" "-p leaf -p mid" + # global inputs force a full run + run_case "Cargo.lock" "$ws" + run_case "Cargo.toml" "$ws" + run_case "base/src/lib.rs .cargo/config.toml" "$ws" + # ignorable and demo-only changes mean nothing to check + run_case "changes/node/added/x README.md .github/workflows/other.yml" "" + run_case "demo/node/src/main.rs" "" + run_case "" "" + # unattributable file: play safe + run_case "mystery.bin" "$ws" + # whole-workspace closure collapses to the --workspace form + run_case "base/src/lib.rs loner/src/lib.rs dev-user/src/lib.rs" "$ws" + echo "self-test OK ($n cases)" >&2 +} + +if [[ "${1:-}" == "--self-test" ]]; then + self_test + exit 0 +fi + +CHANGED_JSON=$(jq -R -s 'split("\n") | map(select(length > 0))') +cargo metadata --no-deps --format-version 1 | scope "$CHANGED_JSON" From c551779b53a2b469b9e9a2264c3511ab1c321608 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Fri, 12 Jun 2026 21:06:08 +0100 Subject: [PATCH 2/4] fix: make it a bit more maintainable Signed-off-by: Giles Cope --- .github/workflows/feature-unification.yml | 7 +- scripts/feature-unification-scope.sh | 285 +++++++++++++++++----- 2 files changed, 228 insertions(+), 64 deletions(-) diff --git a/.github/workflows/feature-unification.yml b/.github/workflows/feature-unification.yml index 11d15d81e..078f3e0ec 100644 --- a/.github/workflows/feature-unification.yml +++ b/.github/workflows/feature-unification.yml @@ -13,7 +13,8 @@ on: paths: - "**/Cargo.toml" - "Cargo.lock" - - "Earthfile" + - "rust-toolchain.toml" + - "scripts/feature-unification-scope.sh" - ".github/workflows/feature-unification.yml" # no top level default permissions for security reasons @@ -50,6 +51,10 @@ jobs: # stable is plenty for `cargo metadata` and saves a 1.95 download. RUSTUP_TOOLCHAIN: stable run: | + # Self-test the scoper, but only when this PR changes it. + if git diff --name-only HEAD^1 HEAD | grep -qx "scripts/feature-unification-scope.sh"; then + ./scripts/feature-unification-scope.sh --self-test + fi packages=$(git diff --name-only HEAD^1 HEAD | ./scripts/feature-unification-scope.sh) echo "scope: ${packages:-}" echo "packages=$packages" >> "$GITHUB_OUTPUT" diff --git a/scripts/feature-unification-scope.sh b/scripts/feature-unification-scope.sh index e8831da03..39405501d 100755 --- a/scripts/feature-unification-scope.sh +++ b/scripts/feature-unification-scope.sh @@ -4,36 +4,137 @@ # Reads changed file paths (one per line) on stdin and prints the package # selection args for `cargo hack check --no-dev-deps`: # -# * "--workspace --exclude ..." a global build input changed, check everything # * "-p a -p b ..." only these crates and their reverse-dependency # closure need re-checking +# * "--workspace --exclude ..." every crate is affected (computed, or a +# global input like rust-toolchain changed) # * "" (empty) nothing compile-relevant changed, skip the check # -# Why the reverse closure is sufficient: the check resolves features -# per-package, so a change in crate X can only alter the outcome for X itself -# or for crates whose dependency graph contains X. Dev-dependency edges are -# ignored on purpose -- the check strips dev-deps, so X can never enter a -# dependent's checked graph through one. +# Scope is the union of two computed sets -- there is no blanket "manifest +# changed, check everything" path: # -# Files inside a crate directory attribute to that crate (even data/markdown: -# they may be include_str!'d). Files that belong to no crate and match no -# IGNORE pattern force a full --workspace run. +# 1. File attribution: each changed file maps to the crate whose directory +# contains it; take the reverse-dependency closure over workspace +# normal/build edges. Dev-dependency edges are ignored on purpose -- the +# check strips dev-deps, so a change can never reach a dependent +# through one. +# 2. Lock diff: if Cargo.lock changed, parse the base and head lock files, +# fingerprint every package by (version, source, checksum, deps), and +# reverse-walk the lock graph from the changed packages to the workspace +# members whose resolution they participate in. If the root Cargo.toml +# changed, dependency names harvested from its diff hunks are added as +# seeds (this catches feature-only [workspace.dependencies] edits, which +# never show up in the lock). +# +# Base for diffs is HEAD^1 (the PR base on a merge-commit checkout); override +# with SCOPE_BASE_REF for local experiments. # # Self-test: scripts/feature-unification-scope.sh --self-test set -euo pipefail -# jq program. stdin: `cargo metadata --no-deps` JSON. $changed: array of paths. +BASE_REF="${SCOPE_BASE_REF:-HEAD^1}" +WORKSPACE_ARGS="--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" + +# Cargo.lock -> JSON array of {name, version, source, checksum, deps:[names]}. +# The lock format is line-regular; names are crates-io-safe ([a-zA-Z0-9_-]), +# so naive JSON assembly is sound. +lock_to_json() { + awk ' + function emit() { + if (name == "") return + printf "{\"name\":\"%s\",\"version\":\"%s\",\"source\":\"%s\",\"checksum\":\"%s\",\"deps\":[%s]}\n", + name, ver, src, cks, deps + } + /^\[\[package\]\]/ { emit(); name=ver=src=cks=deps=""; indeps=0 } + /^name = / { gsub(/"/, ""); name=$3 } + /^version = / { gsub(/"/, ""); ver=$3 } + /^source = / { gsub(/"/, ""); src=$3 } + /^checksum = / { gsub(/"/, ""); cks=$3 } + /^dependencies = \[/ { indeps=1; next } + indeps && /^\]/ { indeps=0 } + indeps { + gsub(/[",]/, "") + # entries are "name" or "name version (source)"; keep the name + if (deps != "") deps = deps "," + deps = deps "\"" $1 "\"" + } + END { emit() } + ' | jq -s '.' +} + +# Names whose locked fingerprint differs between two lock JSONs. The locks +# arrive via --slurpfile (hence [0]): a full lock as JSON is megabytes and +# would overflow ARG_MAX if passed with --argjson. # shellcheck disable=SC2016 # single quotes are deliberate: this is jq, not shell -JQ_PROG=' +JQ_LOCK_CHANGED=' +def fp: group_by(.name) + | map({key: .[0].name, value: (map({version, source, checksum, deps}) | sort)}) + | from_entries; +($old[0] | fp) as $o | ($new[0] | fp) as $n +| [($o | keys[]), ($n | keys[])] | unique +| map(select($o[.] != $n[.])) +' + +# Workspace members ($members) reverse-reachable from $seeds in the lock +# graph ($lock via --slurpfile, see above). +# shellcheck disable=SC2016 +JQ_AFFECTED=' +(reduce $lock[0][] as $p ({}; + reduce ($p.deps[]?) as $d (.; .[$d] = ((.[$d] // []) + [$p.name])))) + as $radj +| {cur: ($seeds | unique), grew: ($seeds | length > 0)} +| until(.grew | not; + . as $s + | ([.cur[] | $radj[.] // []] | add // [] | unique + | map(select(. as $x | $s.cur | index($x) | not))) as $more + | {cur: ((.cur + $more) | sort), grew: ($more | length > 0)}) +| .cur as $reached +| [$members[] | select(. as $m | $reached | index($m))] | sort +' + +# File attribution + workspace reverse closure. stdin: `cargo metadata +# --no-deps` JSON. $changed: paths. $extra: member names seeded by the lock +# diff. Output: the final package-selection arg string. +# shellcheck disable=SC2016 +JQ_ATTRIB=' +# ── helpers ────────────────────────────────────────────────────────────── +def matches_any($regexes): . as $f | any($regexes[]; . as $re | $f | test($re)); + +# The crate whose directory contains the file; longest prefix wins, so +# nested crates (e.g. pallets/x/mock) beat their parent. Null if unowned. +def owning_crate($pkgs): + . as $f + | [$pkgs[] | select((.dir + "/") as $pre | $f | startswith($pre))] + | max_by(.dir | length) + | .name; + +# Grow the input seed array with every package that transitively depends on +# it, walking $deps (name -> [dep names]) in reverse until a fixed point. +def reverse_closure($deps; $names): + {cur: ., grew: (length > 0)} + | until(.grew | not; + . as $s + | ([$names[] | select(. as $n + | ($s.cur | index($n) | not) + and ($deps[$n] | any(. as $d | $s.cur | index($d))))]) as $more + | {cur: ((.cur + $more) | sort), grew: ($more | length > 0)}) + | .cur; + +# ── policy tables ──────────────────────────────────────────────────────── "--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" as $workspace_args | ["partner-chains-demo-node", "partner-chains-demo-runtime"] as $excluded -# A change to any of these invalidates every crate'\''s resolution. -| ["^Cargo\\.toml$", "^Cargo\\.lock$", "^\\.cargo/", "^\\.config/", - "^rust-toolchain", "^Earthfile$", "^scripts/feature-unification-scope\\.sh$", - "^\\.github/workflows/feature-unification\\.yml$"] as $global +# Global inputs with no diffable crate mapping: toolchain and cargo config. +| ["^\\.cargo/", "^\\.config/", "^rust-toolchain"] as $global +# Handled out-of-band by the lock/manifest diff in the wrapper script. +| ["^Cargo\\.toml$", "^Cargo\\.lock$"] as $handled # Never compile-relevant (only consulted for files outside every crate dir). -| ["^changes/", "^\\.changes_archive/", "^\\.github/", "\\.md$", "^LICENSE"] as $ignore +# Earthfile and this scoper are deliberately here: build-recipe or scoper +# edits should not force a full workspace re-check. +| ["^changes/", "^\\.changes_archive/", "^\\.github/", "\\.md$", "^LICENSE", + "^Earthfile$", "^scripts/feature-unification-scope\\.sh$"] as $ignore + +# ── workspace shape, from cargo metadata on stdin ──────────────────────── | .workspace_root as $root | [.packages[] | {name, dir: (.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr($root + "/"))}] as $pkgs @@ -44,46 +145,60 @@ JQ_PROG=' | select((.kind == null or .kind == "build") and (.name as $n | $names | index($n))) | .name])) as $deps -| if any($changed[]; . as $f | any($global[]; . as $re | $f | test($re))) then + +# ── 1. sort the changed files into owned / ignored / unattributable ────── +| ([$changed[] | select(matches_any($handled) | not)]) as $files +| ([$files[] | owning_crate($pkgs) | select(. != null)]) as $touched +| ([$files[] | select((owning_crate($pkgs) == null) and (matches_any($ignore) | not))]) + as $unattributable + +# ── 2. decide the scope ────────────────────────────────────────────────── +| if any($changed[]; matches_any($global)) or ($unattributable != []) then $workspace_args else - # Attribute each file to the crate with the longest matching dir prefix, - # so nested crates (e.g. pallets/x/mock) win over their parent. - ([$changed[] | . as $f - | ([$pkgs[] | select((.dir + "/") as $pre | $f | startswith($pre))] | max_by(.dir | length)) - | if . != null then {pkg: .name} - elif any($ignore[]; . as $re | $f | test($re)) then empty - else {unattributable: $f} - end]) - as $hits - | if any($hits[]; .unattributable) then $workspace_args - else - ([$hits[].pkg] | unique) as $seed - # Reverse-dependency closure: grow until no new dependent is added. - | {cur: $seed, grew: ($seed | length > 0)} - | until(.grew | not; - . as $s - | ([$names[] | select(. as $n - | ($s.cur | index($n) | not) - and ($deps[$n] | any(. as $d | $s.cur | index($d))))]) - as $more - | {cur: ((.cur + $more) | sort), grew: ($more | length > 0)}) - | (.cur - $excluded) as $closure - | if $closure == [] then "" - elif $closure == (($names - $excluded) | sort) then $workspace_args - else [$closure[] | "-p " + .] | join(" ") - end + ((($touched + $extra) | unique) | reverse_closure($deps; $names)) - $excluded + | if . == [] then "" + elif . == (($names - $excluded) | sort) then $workspace_args + else [.[] | "-p " + .] | join(" ") end end ' -scope() { # $1: changed paths as JSON array; metadata JSON on stdin - jq -r --argjson changed "$1" "$JQ_PROG" +scope() { # $1: changed paths JSON array; $2: extra member seeds JSON array; metadata on stdin + jq -r --argjson changed "$1" --argjson extra "$2" "$JQ_ATTRIB" +} + +# Member names whose resolution changed between the base and head lock files. +# $1: changed paths JSON array; $2: members JSON array. +lock_affected() { + local seeds="[]" toml_seeds tmpd + tmpd=$(mktemp -d) + # shellcheck disable=SC2064 # expand $tmpd now, not at trap time + trap "rm -rf '$tmpd'" RETURN + lock_to_json "$tmpd/new.json" + if jq -e 'index("Cargo.lock")' >/dev/null <<<"$1"; then + if ! git cat-file -e "$BASE_REF:Cargo.lock" 2>/dev/null; then + return 1 # no base lock to diff against: caller falls back to full + fi + git show "$BASE_REF:Cargo.lock" | lock_to_json >"$tmpd/old.json" + seeds=$(jq -n --slurpfile old "$tmpd/old.json" \ + --slurpfile new "$tmpd/new.json" "$JQ_LOCK_CHANGED") + fi + if jq -e 'index("Cargo.toml")' >/dev/null <<<"$1"; then + # Dep names from changed lines of the root manifest; tokens that are + # not package names simply match nothing in the lock graph. + toml_seeds=$(git diff "$BASE_REF" HEAD -- Cargo.toml 2>/dev/null | + sed -n 's/^[+-][[:space:]]*\([a-zA-Z0-9_-]\{1,\}\)[[:space:]]*=.*/\1/p' | + jq -R -s 'split("\n") | map(select(length > 0)) | unique') + seeds=$(jq -n --argjson a "$seeds" --argjson b "${toml_seeds:-[]}" '$a + $b | unique') + fi + jq -n --slurpfile lock "$tmpd/new.json" \ + --argjson members "$2" --argjson seeds "$seeds" "$JQ_AFFECTED" } self_test() { local meta ws got n=0 - ws="--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" + ws="$WORKSPACE_ARGS" # base <- mid <- leaf; dev-user dev-depends on base; loner is isolated; # demo crates are the excluded ones. meta='{ @@ -103,35 +218,66 @@ self_test() { "dependencies": []} ] }' - run_case() { # $1: changed files (space-sep), $2: expected output + run_case() { # $1: changed files (space-sep), $2: extra seeds JSON, $3: expected local files_json # shellcheck disable=SC2086 # splitting $1 into one path per line is the point files_json=$(printf '%s\n' $1 | jq -R -s 'split("\n") | map(select(length > 0))') - got=$(echo "$meta" | scope "$files_json") - if [[ "$got" != "$2" ]]; then - echo "self-test FAIL for [$1]: expected '$2', got '$got'" >&2 + got=$(echo "$meta" | scope "$files_json" "$2") + if [[ "$got" != "$3" ]]; then + echo "self-test FAIL for [$1|$2]: expected '$3', got '$got'" >&2 exit 1 fi n=$((n + 1)) } # change in base ripples up through mid to leaf; not to the dev-only # user, the loner, or the excluded demo crate - run_case "base/src/lib.rs" "-p base -p leaf -p mid" - run_case "leaf/src/lib.rs" "-p leaf" + run_case "base/src/lib.rs" "[]" "-p base -p leaf -p mid" + run_case "leaf/src/lib.rs" "[]" "-p leaf" # crate manifest change scopes like a source change - run_case "mid/Cargo.toml" "-p leaf -p mid" - # global inputs force a full run - run_case "Cargo.lock" "$ws" - run_case "Cargo.toml" "$ws" - run_case "base/src/lib.rs .cargo/config.toml" "$ws" + run_case "mid/Cargo.toml" "[]" "-p leaf -p mid" + # global inputs with no crate mapping force a full run + run_case "rust-toolchain.toml" "[]" "$ws" + run_case "base/src/lib.rs .cargo/config.toml" "[]" "$ws" # ignorable and demo-only changes mean nothing to check - run_case "changes/node/added/x README.md .github/workflows/other.yml" "" - run_case "demo/node/src/main.rs" "" - run_case "" "" + run_case "changes/node/added/x README.md .github/workflows/other.yml" "[]" "" + run_case "Earthfile scripts/feature-unification-scope.sh" "[]" "" + run_case "demo/node/src/main.rs" "[]" "" + run_case "" "[]" "" # unattributable file: play safe - run_case "mystery.bin" "$ws" + run_case "mystery.bin" "[]" "$ws" # whole-workspace closure collapses to the --workspace form - run_case "base/src/lib.rs loner/src/lib.rs dev-user/src/lib.rs" "$ws" + run_case "base/src/lib.rs loner/src/lib.rs dev-user/src/lib.rs" "[]" "$ws" + # lock-diff seeds merge with file attribution (root manifests themselves + # are handled out-of-band, hence no fallback here) + run_case "Cargo.lock" '["mid"]' "-p leaf -p mid" + run_case "Cargo.toml Cargo.lock" "[]" "" + run_case "leaf/src/lib.rs Cargo.lock" '["loner"]' "-p leaf -p loner" + + # lock fingerprint diff: version bump of extdep, member mid's deps change + local old new changed affected + old='[{"name":"extdep","version":"1.0.0","source":"reg","checksum":"a","deps":[]}, + {"name":"mid","version":"0.1.0","source":"","checksum":"","deps":["extdep"]}, + {"name":"leaf","version":"0.1.0","source":"","checksum":"","deps":["mid"]}, + {"name":"loner","version":"0.1.0","source":"","checksum":"","deps":[]}]' + new=${old/1.0.0/1.1.0} + tmpd=$(mktemp -d) + echo "$old" >"$tmpd/old.json"; echo "$new" >"$tmpd/new.json" + changed=$(jq -n --slurpfile old "$tmpd/old.json" --slurpfile new "$tmpd/new.json" "$JQ_LOCK_CHANGED") + [[ "$(jq -c . <<<"$changed")" == '["extdep"]' ]] || + { echo "self-test FAIL: lock diff expected [\"extdep\"], got $changed" >&2; exit 1; } + n=$((n + 1)) + # reverse reach: extdep -> mid -> leaf, but never loner + affected=$(jq -n --slurpfile lock "$tmpd/new.json" --argjson members '["mid","leaf","loner"]' \ + --argjson seeds "$changed" "$JQ_AFFECTED") + [[ "$(jq -c . <<<"$affected")" == '["leaf","mid"]' ]] || + { echo "self-test FAIL: affected expected [\"leaf\",\"mid\"], got $affected" >&2; exit 1; } + n=$((n + 1)) + # identical locks: nothing changed + changed=$(jq -n --slurpfile old "$tmpd/old.json" --slurpfile new "$tmpd/old.json" "$JQ_LOCK_CHANGED") + rm -rf "$tmpd" + [[ "$(jq -c . <<<"$changed")" == '[]' ]] || + { echo "self-test FAIL: identical locks expected [], got $changed" >&2; exit 1; } + n=$((n + 1)) echo "self-test OK ($n cases)" >&2 } @@ -141,4 +287,17 @@ if [[ "${1:-}" == "--self-test" ]]; then fi CHANGED_JSON=$(jq -R -s 'split("\n") | map(select(length > 0))') -cargo metadata --no-deps --format-version 1 | scope "$CHANGED_JSON" +METADATA=$(cargo metadata --no-deps --format-version 1) +MEMBERS=$(jq '[.packages[].name]' <<<"$METADATA") + +EXTRA="[]" +if jq -e 'index("Cargo.lock") or index("Cargo.toml")' >/dev/null <<<"$CHANGED_JSON"; then + if ! EXTRA=$(lock_affected "$CHANGED_JSON" "$MEMBERS"); then + # No base lock to diff against (e.g. shallow clone surprise): the one + # remaining conservative fallback. + echo "$WORKSPACE_ARGS" + exit 0 + fi +fi + +scope "$CHANGED_JSON" "$EXTRA" <<<"$METADATA" From 21913457ad41cb8368795e2488640088d0b0d9ee Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Mon, 15 Jun 2026 18:02:17 +0100 Subject: [PATCH 3/4] chore: switch from JQ to TS Signed-off-by: Giles Cope --- .github/dependabot.yml | 7 + .github/workflows/feature-unification.yml | 33 +-- .gitignore | 3 + Earthfile | 46 +++- scripts/feature-unification-scope.sh | 303 ---------------------- scripts/feature-unification-scope.ts | 270 +++++++++++++++++++ scripts/package-lock.json | 27 ++ scripts/package.json | 10 + 8 files changed, 366 insertions(+), 333 deletions(-) delete mode 100755 scripts/feature-unification-scope.sh create mode 100644 scripts/feature-unification-scope.ts create mode 100644 scripts/package-lock.json create mode 100644 scripts/package.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml index da20f8fd4..3812c6e7f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -23,6 +23,13 @@ updates: cooldown: default-days: 7 + - package-ecosystem: "npm" + directory: "/scripts" + schedule: + interval: "daily" + cooldown: + default-days: 7 + - package-ecosystem: "docker" directory: "/" schedule: diff --git a/.github/workflows/feature-unification.yml b/.github/workflows/feature-unification.yml index 078f3e0ec..710a94fdf 100644 --- a/.github/workflows/feature-unification.yml +++ b/.github/workflows/feature-unification.yml @@ -14,7 +14,7 @@ on: - "**/Cargo.toml" - "Cargo.lock" - "rust-toolchain.toml" - - "scripts/feature-unification-scope.sh" + - "scripts/feature-unification-scope.ts" - ".github/workflows/feature-unification.yml" # no top level default permissions for security reasons @@ -42,27 +42,10 @@ jobs: - id: guard uses: ./.github/actions/tree-cache-guard - # Only check the reverse-dependency closure of the crates the PR diff - # touches (see scripts/feature-unification-scope.sh). - - id: scope - if: steps.guard.outputs.hit != 'true' - env: - # Skip the rust-toolchain.toml pin: the runner's preinstalled - # stable is plenty for `cargo metadata` and saves a 1.95 download. - RUSTUP_TOOLCHAIN: stable - run: | - # Self-test the scoper, but only when this PR changes it. - if git diff --name-only HEAD^1 HEAD | grep -qx "scripts/feature-unification-scope.sh"; then - ./scripts/feature-unification-scope.sh --self-test - fi - packages=$(git diff --name-only HEAD^1 HEAD | ./scripts/feature-unification-scope.sh) - echo "scope: ${packages:-}" - echo "packages=$packages" >> "$GITHUB_OUTPUT" - # No GHCR login: the build chain only pulls the public # midnightntwrk/midnight-node-ci image from docker.io. - uses: EarthBuild/actions-setup@cae2d9ab68894d8402751fe42e07c7cca0272f7f - if: steps.guard.outputs.hit != 'true' && steps.scope.outputs.packages != '' + if: steps.guard.outputs.hit != 'true' with: version: v0.8.16 github-token: ${{ github.token }} @@ -70,15 +53,17 @@ jobs: # Uncomment if disk space again becomes an issue. # - name: Free disk space - # if: steps.guard.outputs.hit != 'true' && steps.scope.outputs.packages != '' + # if: steps.guard.outputs.hit != 'true' # run: scripts/free-disk-space.sh + # Scope is computed inside the earthly target now (the + # +feature-unification-inputs LOCALLY target does the only git work, then + # scripts/feature-unification-scope.ts attributes the diff in-container). + # An empty scope skips the cargo-hack check from within the target. - name: Run feature unification check - if: steps.guard.outputs.hit != 'true' && steps.scope.outputs.packages != '' - env: - PACKAGES: ${{ steps.scope.outputs.packages }} + if: steps.guard.outputs.hit != 'true' run: | - . ./.envrc && earthly --ci +check-feature-unification --PACKAGES="$PACKAGES" + . ./.envrc && earthly --ci +check-feature-unification - uses: ./.github/actions/tree-cache-guard/save if: steps.guard.outputs.hit != 'true' diff --git a/.gitignore b/.gitignore index 5e283de7c..de0fdb76d 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,6 @@ AGENTS.local.md # Local-only image bundle for +local-env-ci-localimg (docker save → load); never commit local-env-images.tar toolkit-image.tar + +# Scratch dir written by the +feature-unification-inputs LOCALLY target +.scope/ diff --git a/Earthfile b/Earthfile index 950ad0c99..34f852c3f 100644 --- a/Earthfile +++ b/Earthfile @@ -854,29 +854,63 @@ check-rust: ENV SKIP_WASM_BUILD=1 +# feature-unification-inputs is the entire host footprint of the scope step: +# three text files the scoper reads, all derived from git history (which only +# exists on the host, not in a container). HEAD^1 is the PR base on a +# merge-commit checkout; override with --SCOPE_BASE_REF for local runs. +feature-unification-inputs: + LOCALLY + ARG SCOPE_BASE_REF=HEAD^1 + RUN mkdir -p .scope && \ + git diff --name-only "$SCOPE_BASE_REF" HEAD > .scope/changed.txt && \ + { git show "$SCOPE_BASE_REF:Cargo.lock" > .scope/base-lock.txt 2>/dev/null || : > .scope/base-lock.txt; } && \ + { git diff "$SCOPE_BASE_REF" HEAD -- Cargo.toml > .scope/toml-diff.txt 2>/dev/null || : > .scope/toml-diff.txt; } + SAVE ARTIFACT .scope/changed.txt changed.txt + SAVE ARTIFACT .scope/base-lock.txt base-lock.txt + SAVE ARTIFACT .scope/toml-diff.txt toml-diff.txt + # check-feature-unification verifies each crate compiles without dev-deps, # catching issues where workspace feature unification masks missing dependencies. # The partner-chains demo crates are excluded: they are upstream examples, not # shipped artifacts, and cost ~5min of the serial check. +# +# Scope is computed in-container by scripts/feature-unification-scope.ts (the +# reverse-dependency closure of the crates the PR diff touches) so the only +# host work is the git reads in +feature-unification-inputs. An empty scope +# (nothing compile-relevant changed) skips the check entirely. check-feature-unification: FROM +check-rust-prepare CACHE --sharing shared --id cargo-git /usr/local/cargo/git CACHE --sharing shared --id cargo-reg /usr/local/cargo/registry + # Scope tooling deps (smol-toml) in their own layer so workspace edits don't + # reinstall. node + npm are pinned in the CI base image. + COPY scripts/package.json scripts/package-lock.json scripts/ + RUN cd scripts && npm ci --no-audit --no-fund COPY --keep-ts --dir \ Cargo.lock Cargo.toml .config .sqlx deny.toml docs \ ledger LICENSE node pallets primitives README.md res runtime \ metadata rustfmt.toml util tests relay partner-chains COMPACTC_VERSION . + COPY scripts/feature-unification-scope.ts scripts/feature-unification-scope.ts + COPY +feature-unification-inputs/changed.txt \ + +feature-unification-inputs/base-lock.txt \ + +feature-unification-inputs/toml-diff.txt .scope/ ENV SKIP_WASM_BUILD=1 ENV CARGO_INCREMENTAL=0 - # Package scope: full workspace by default; PR builds pass the output of - # scripts/feature-unification-scope.sh (reverse-dependency closure of the - # crates touched by the diff) to skip re-checking unaffected crates. - ARG PACKAGES="--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" # Pinned: an unpinned binstall here can drift from the version baked into # the CI base image and change check behaviour between runs. - RUN cargo binstall --no-confirm --locked cargo-hack@0.6.45 - RUN cargo hack check $PACKAGES --no-dev-deps + # renovate: datasource=crate packageName=cargo-hack + ARG CARGO_HACK_VERSION=0.6.45 + RUN cargo binstall --no-confirm --locked cargo-hack@${CARGO_HACK_VERSION} + # node is pinned in the CI base image; the scoper reads the git-derived + # inputs and emits the `-p` selection (empty => nothing to check). + RUN PACKAGES="$(node scripts/feature-unification-scope.ts \ + .scope/changed.txt .scope/base-lock.txt .scope/toml-diff.txt)" && \ + if [ -z "$PACKAGES" ]; then \ + echo "feature-unification: nothing affected — skipping"; exit 0; \ + fi && \ + echo "feature-unification scope: $PACKAGES" && \ + cargo hack check $PACKAGES --no-dev-deps # check-metadata confirms that metadata in the repo matches a given node image check-metadata: diff --git a/scripts/feature-unification-scope.sh b/scripts/feature-unification-scope.sh deleted file mode 100755 index 39405501d..000000000 --- a/scripts/feature-unification-scope.sh +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env bash -# Compute the cargo-hack package scope for the feature-unification check. -# -# Reads changed file paths (one per line) on stdin and prints the package -# selection args for `cargo hack check --no-dev-deps`: -# -# * "-p a -p b ..." only these crates and their reverse-dependency -# closure need re-checking -# * "--workspace --exclude ..." every crate is affected (computed, or a -# global input like rust-toolchain changed) -# * "" (empty) nothing compile-relevant changed, skip the check -# -# Scope is the union of two computed sets -- there is no blanket "manifest -# changed, check everything" path: -# -# 1. File attribution: each changed file maps to the crate whose directory -# contains it; take the reverse-dependency closure over workspace -# normal/build edges. Dev-dependency edges are ignored on purpose -- the -# check strips dev-deps, so a change can never reach a dependent -# through one. -# 2. Lock diff: if Cargo.lock changed, parse the base and head lock files, -# fingerprint every package by (version, source, checksum, deps), and -# reverse-walk the lock graph from the changed packages to the workspace -# members whose resolution they participate in. If the root Cargo.toml -# changed, dependency names harvested from its diff hunks are added as -# seeds (this catches feature-only [workspace.dependencies] edits, which -# never show up in the lock). -# -# Base for diffs is HEAD^1 (the PR base on a merge-commit checkout); override -# with SCOPE_BASE_REF for local experiments. -# -# Self-test: scripts/feature-unification-scope.sh --self-test -set -euo pipefail - -BASE_REF="${SCOPE_BASE_REF:-HEAD^1}" -WORKSPACE_ARGS="--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" - -# Cargo.lock -> JSON array of {name, version, source, checksum, deps:[names]}. -# The lock format is line-regular; names are crates-io-safe ([a-zA-Z0-9_-]), -# so naive JSON assembly is sound. -lock_to_json() { - awk ' - function emit() { - if (name == "") return - printf "{\"name\":\"%s\",\"version\":\"%s\",\"source\":\"%s\",\"checksum\":\"%s\",\"deps\":[%s]}\n", - name, ver, src, cks, deps - } - /^\[\[package\]\]/ { emit(); name=ver=src=cks=deps=""; indeps=0 } - /^name = / { gsub(/"/, ""); name=$3 } - /^version = / { gsub(/"/, ""); ver=$3 } - /^source = / { gsub(/"/, ""); src=$3 } - /^checksum = / { gsub(/"/, ""); cks=$3 } - /^dependencies = \[/ { indeps=1; next } - indeps && /^\]/ { indeps=0 } - indeps { - gsub(/[",]/, "") - # entries are "name" or "name version (source)"; keep the name - if (deps != "") deps = deps "," - deps = deps "\"" $1 "\"" - } - END { emit() } - ' | jq -s '.' -} - -# Names whose locked fingerprint differs between two lock JSONs. The locks -# arrive via --slurpfile (hence [0]): a full lock as JSON is megabytes and -# would overflow ARG_MAX if passed with --argjson. -# shellcheck disable=SC2016 # single quotes are deliberate: this is jq, not shell -JQ_LOCK_CHANGED=' -def fp: group_by(.name) - | map({key: .[0].name, value: (map({version, source, checksum, deps}) | sort)}) - | from_entries; -($old[0] | fp) as $o | ($new[0] | fp) as $n -| [($o | keys[]), ($n | keys[])] | unique -| map(select($o[.] != $n[.])) -' - -# Workspace members ($members) reverse-reachable from $seeds in the lock -# graph ($lock via --slurpfile, see above). -# shellcheck disable=SC2016 -JQ_AFFECTED=' -(reduce $lock[0][] as $p ({}; - reduce ($p.deps[]?) as $d (.; .[$d] = ((.[$d] // []) + [$p.name])))) - as $radj -| {cur: ($seeds | unique), grew: ($seeds | length > 0)} -| until(.grew | not; - . as $s - | ([.cur[] | $radj[.] // []] | add // [] | unique - | map(select(. as $x | $s.cur | index($x) | not))) as $more - | {cur: ((.cur + $more) | sort), grew: ($more | length > 0)}) -| .cur as $reached -| [$members[] | select(. as $m | $reached | index($m))] | sort -' - -# File attribution + workspace reverse closure. stdin: `cargo metadata -# --no-deps` JSON. $changed: paths. $extra: member names seeded by the lock -# diff. Output: the final package-selection arg string. -# shellcheck disable=SC2016 -JQ_ATTRIB=' -# ── helpers ────────────────────────────────────────────────────────────── -def matches_any($regexes): . as $f | any($regexes[]; . as $re | $f | test($re)); - -# The crate whose directory contains the file; longest prefix wins, so -# nested crates (e.g. pallets/x/mock) beat their parent. Null if unowned. -def owning_crate($pkgs): - . as $f - | [$pkgs[] | select((.dir + "/") as $pre | $f | startswith($pre))] - | max_by(.dir | length) - | .name; - -# Grow the input seed array with every package that transitively depends on -# it, walking $deps (name -> [dep names]) in reverse until a fixed point. -def reverse_closure($deps; $names): - {cur: ., grew: (length > 0)} - | until(.grew | not; - . as $s - | ([$names[] | select(. as $n - | ($s.cur | index($n) | not) - and ($deps[$n] | any(. as $d | $s.cur | index($d))))]) as $more - | {cur: ((.cur + $more) | sort), grew: ($more | length > 0)}) - | .cur; - -# ── policy tables ──────────────────────────────────────────────────────── -"--workspace --exclude partner-chains-demo-node --exclude partner-chains-demo-runtime" - as $workspace_args -| ["partner-chains-demo-node", "partner-chains-demo-runtime"] as $excluded -# Global inputs with no diffable crate mapping: toolchain and cargo config. -| ["^\\.cargo/", "^\\.config/", "^rust-toolchain"] as $global -# Handled out-of-band by the lock/manifest diff in the wrapper script. -| ["^Cargo\\.toml$", "^Cargo\\.lock$"] as $handled -# Never compile-relevant (only consulted for files outside every crate dir). -# Earthfile and this scoper are deliberately here: build-recipe or scoper -# edits should not force a full workspace re-check. -| ["^changes/", "^\\.changes_archive/", "^\\.github/", "\\.md$", "^LICENSE", - "^Earthfile$", "^scripts/feature-unification-scope\\.sh$"] as $ignore - -# ── workspace shape, from cargo metadata on stdin ──────────────────────── -| .workspace_root as $root -| [.packages[] | {name, dir: (.manifest_path | rtrimstr("/Cargo.toml") | ltrimstr($root + "/"))}] - as $pkgs -| ($pkgs | map(.name)) as $names -# name -> [workspace deps], normal/build kinds only (see header for why not dev) -| (reduce .packages[] as $p ({}; - .[$p.name] = [$p.dependencies[] - | select((.kind == null or .kind == "build") and (.name as $n | $names | index($n))) - | .name])) - as $deps - -# ── 1. sort the changed files into owned / ignored / unattributable ────── -| ([$changed[] | select(matches_any($handled) | not)]) as $files -| ([$files[] | owning_crate($pkgs) | select(. != null)]) as $touched -| ([$files[] | select((owning_crate($pkgs) == null) and (matches_any($ignore) | not))]) - as $unattributable - -# ── 2. decide the scope ────────────────────────────────────────────────── -| if any($changed[]; matches_any($global)) or ($unattributable != []) then - $workspace_args - else - ((($touched + $extra) | unique) | reverse_closure($deps; $names)) - $excluded - | if . == [] then "" - elif . == (($names - $excluded) | sort) then $workspace_args - else [.[] | "-p " + .] | join(" ") - end - end -' - -scope() { # $1: changed paths JSON array; $2: extra member seeds JSON array; metadata on stdin - jq -r --argjson changed "$1" --argjson extra "$2" "$JQ_ATTRIB" -} - -# Member names whose resolution changed between the base and head lock files. -# $1: changed paths JSON array; $2: members JSON array. -lock_affected() { - local seeds="[]" toml_seeds tmpd - tmpd=$(mktemp -d) - # shellcheck disable=SC2064 # expand $tmpd now, not at trap time - trap "rm -rf '$tmpd'" RETURN - lock_to_json "$tmpd/new.json" - if jq -e 'index("Cargo.lock")' >/dev/null <<<"$1"; then - if ! git cat-file -e "$BASE_REF:Cargo.lock" 2>/dev/null; then - return 1 # no base lock to diff against: caller falls back to full - fi - git show "$BASE_REF:Cargo.lock" | lock_to_json >"$tmpd/old.json" - seeds=$(jq -n --slurpfile old "$tmpd/old.json" \ - --slurpfile new "$tmpd/new.json" "$JQ_LOCK_CHANGED") - fi - if jq -e 'index("Cargo.toml")' >/dev/null <<<"$1"; then - # Dep names from changed lines of the root manifest; tokens that are - # not package names simply match nothing in the lock graph. - toml_seeds=$(git diff "$BASE_REF" HEAD -- Cargo.toml 2>/dev/null | - sed -n 's/^[+-][[:space:]]*\([a-zA-Z0-9_-]\{1,\}\)[[:space:]]*=.*/\1/p' | - jq -R -s 'split("\n") | map(select(length > 0)) | unique') - seeds=$(jq -n --argjson a "$seeds" --argjson b "${toml_seeds:-[]}" '$a + $b | unique') - fi - jq -n --slurpfile lock "$tmpd/new.json" \ - --argjson members "$2" --argjson seeds "$seeds" "$JQ_AFFECTED" -} - -self_test() { - local meta ws got n=0 - ws="$WORKSPACE_ARGS" - # base <- mid <- leaf; dev-user dev-depends on base; loner is isolated; - # demo crates are the excluded ones. - meta='{ - "workspace_root": "/ws", - "packages": [ - {"name": "base", "manifest_path": "/ws/base/Cargo.toml", "dependencies": []}, - {"name": "mid", "manifest_path": "/ws/mid/Cargo.toml", - "dependencies": [{"name": "base", "kind": null}]}, - {"name": "leaf", "manifest_path": "/ws/leaf/Cargo.toml", - "dependencies": [{"name": "mid", "kind": null}]}, - {"name": "loner", "manifest_path": "/ws/loner/Cargo.toml", "dependencies": []}, - {"name": "dev-user", "manifest_path": "/ws/dev-user/Cargo.toml", - "dependencies": [{"name": "base", "kind": "dev"}]}, - {"name": "partner-chains-demo-node", "manifest_path": "/ws/demo/node/Cargo.toml", - "dependencies": [{"name": "base", "kind": null}]}, - {"name": "partner-chains-demo-runtime", "manifest_path": "/ws/demo/runtime/Cargo.toml", - "dependencies": []} - ] - }' - run_case() { # $1: changed files (space-sep), $2: extra seeds JSON, $3: expected - local files_json - # shellcheck disable=SC2086 # splitting $1 into one path per line is the point - files_json=$(printf '%s\n' $1 | jq -R -s 'split("\n") | map(select(length > 0))') - got=$(echo "$meta" | scope "$files_json" "$2") - if [[ "$got" != "$3" ]]; then - echo "self-test FAIL for [$1|$2]: expected '$3', got '$got'" >&2 - exit 1 - fi - n=$((n + 1)) - } - # change in base ripples up through mid to leaf; not to the dev-only - # user, the loner, or the excluded demo crate - run_case "base/src/lib.rs" "[]" "-p base -p leaf -p mid" - run_case "leaf/src/lib.rs" "[]" "-p leaf" - # crate manifest change scopes like a source change - run_case "mid/Cargo.toml" "[]" "-p leaf -p mid" - # global inputs with no crate mapping force a full run - run_case "rust-toolchain.toml" "[]" "$ws" - run_case "base/src/lib.rs .cargo/config.toml" "[]" "$ws" - # ignorable and demo-only changes mean nothing to check - run_case "changes/node/added/x README.md .github/workflows/other.yml" "[]" "" - run_case "Earthfile scripts/feature-unification-scope.sh" "[]" "" - run_case "demo/node/src/main.rs" "[]" "" - run_case "" "[]" "" - # unattributable file: play safe - run_case "mystery.bin" "[]" "$ws" - # whole-workspace closure collapses to the --workspace form - run_case "base/src/lib.rs loner/src/lib.rs dev-user/src/lib.rs" "[]" "$ws" - # lock-diff seeds merge with file attribution (root manifests themselves - # are handled out-of-band, hence no fallback here) - run_case "Cargo.lock" '["mid"]' "-p leaf -p mid" - run_case "Cargo.toml Cargo.lock" "[]" "" - run_case "leaf/src/lib.rs Cargo.lock" '["loner"]' "-p leaf -p loner" - - # lock fingerprint diff: version bump of extdep, member mid's deps change - local old new changed affected - old='[{"name":"extdep","version":"1.0.0","source":"reg","checksum":"a","deps":[]}, - {"name":"mid","version":"0.1.0","source":"","checksum":"","deps":["extdep"]}, - {"name":"leaf","version":"0.1.0","source":"","checksum":"","deps":["mid"]}, - {"name":"loner","version":"0.1.0","source":"","checksum":"","deps":[]}]' - new=${old/1.0.0/1.1.0} - tmpd=$(mktemp -d) - echo "$old" >"$tmpd/old.json"; echo "$new" >"$tmpd/new.json" - changed=$(jq -n --slurpfile old "$tmpd/old.json" --slurpfile new "$tmpd/new.json" "$JQ_LOCK_CHANGED") - [[ "$(jq -c . <<<"$changed")" == '["extdep"]' ]] || - { echo "self-test FAIL: lock diff expected [\"extdep\"], got $changed" >&2; exit 1; } - n=$((n + 1)) - # reverse reach: extdep -> mid -> leaf, but never loner - affected=$(jq -n --slurpfile lock "$tmpd/new.json" --argjson members '["mid","leaf","loner"]' \ - --argjson seeds "$changed" "$JQ_AFFECTED") - [[ "$(jq -c . <<<"$affected")" == '["leaf","mid"]' ]] || - { echo "self-test FAIL: affected expected [\"leaf\",\"mid\"], got $affected" >&2; exit 1; } - n=$((n + 1)) - # identical locks: nothing changed - changed=$(jq -n --slurpfile old "$tmpd/old.json" --slurpfile new "$tmpd/old.json" "$JQ_LOCK_CHANGED") - rm -rf "$tmpd" - [[ "$(jq -c . <<<"$changed")" == '[]' ]] || - { echo "self-test FAIL: identical locks expected [], got $changed" >&2; exit 1; } - n=$((n + 1)) - echo "self-test OK ($n cases)" >&2 -} - -if [[ "${1:-}" == "--self-test" ]]; then - self_test - exit 0 -fi - -CHANGED_JSON=$(jq -R -s 'split("\n") | map(select(length > 0))') -METADATA=$(cargo metadata --no-deps --format-version 1) -MEMBERS=$(jq '[.packages[].name]' <<<"$METADATA") - -EXTRA="[]" -if jq -e 'index("Cargo.lock") or index("Cargo.toml")' >/dev/null <<<"$CHANGED_JSON"; then - if ! EXTRA=$(lock_affected "$CHANGED_JSON" "$MEMBERS"); then - # No base lock to diff against (e.g. shallow clone surprise): the one - # remaining conservative fallback. - echo "$WORKSPACE_ARGS" - exit 0 - fi -fi - -scope "$CHANGED_JSON" "$EXTRA" <<<"$METADATA" diff --git a/scripts/feature-unification-scope.ts b/scripts/feature-unification-scope.ts new file mode 100644 index 000000000..7c81e6f0e --- /dev/null +++ b/scripts/feature-unification-scope.ts @@ -0,0 +1,270 @@ +#!/usr/bin/env node +// Compute the cargo-hack package scope for the feature-unification check. +// +// Usage: +// node feature-unification-scope.ts +// +// file: PR-changed paths, one per line (git diff --name-only) +// file: the base commit's Cargo.lock (empty if none) +// file: `git diff` of the root Cargo.toml (empty if untouched) +// +// All git access lives in the Earthfile's `feature-unification-inputs` LOCALLY +// target, which produces these three files; this script is pure computation +// over them plus `cargo metadata` and the head `Cargo.lock` (both read from the +// current workspace, i.e. inside the check container). Prints the package +// selection args for `cargo hack check --no-dev-deps`: +// +// * "-p a -p b ..." only these crates and their reverse-dependency +// closure need re-checking +// * "--workspace --exclude ..." every crate is affected (computed, or a +// global input like rust-toolchain changed) +// * "" (empty) nothing compile-relevant changed, skip the check +// +// Scope is the union of two computed sets -- there is no blanket "manifest +// changed, check everything" path: +// +// 1. File attribution: each changed file maps to the crate whose directory +// contains it; take the reverse-dependency closure over workspace +// normal/build edges. Dev-dependency edges are ignored on purpose -- the +// check strips dev-deps, so a change can never reach a dependent through +// one. +// 2. Lock diff: if Cargo.lock changed, fingerprint every package in the base +// and head locks by (version, source, checksum, deps) and reverse-walk the +// lock graph from the changed packages to the workspace members whose +// resolution they participate in. If the root Cargo.toml changed, dep names +// harvested from its diff hunks are added as seeds (this catches +// feature-only [workspace.dependencies] edits, which never touch the lock). +// +// Runs on Node >= 22.18 (native TypeScript type stripping), no build step. +// Deps: scripts/package.json (smol-toml); `npm ci` in the check target. The CI +// image pins node, so the result is reproducible. + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { parse as parseToml } from "smol-toml"; + +const EXCLUDED = ["partner-chains-demo-node", "partner-chains-demo-runtime"]; +const WORKSPACE_ARGS = `--workspace ${EXCLUDED.map((e) => `--exclude ${e}`).join(" ")}`; +const MAX_BUFFER = 512 * 1024 * 1024; // cargo metadata can be MBs + +// Global inputs with no diffable crate mapping: toolchain and cargo config. +const GLOBAL = [/^\.cargo\//, /^\.config\//, /^rust-toolchain/]; +// Handled out-of-band by the lock/manifest diff below (root manifests only). +const HANDLED = [/^Cargo\.toml$/, /^Cargo\.lock$/]; +// Never compile-relevant (only consulted for files outside every crate dir). +// Earthfile and this scoper are deliberately here: build-recipe or scoper +// edits should not force a full workspace re-check. +const IGNORE = [ + /^changes\//, + /^\.changes_archive\//, + /^\.github\//, + /\.md$/, + /^LICENSE/, + /^Earthfile$/, + /^scripts\/feature-unification-scope\.ts$/, +]; + +interface LockPkg { + name: string; + version: string; + source: string; + checksum: string; + deps: string[]; +} +interface MetaPkg { + name: string; + manifest_path: string; + dependencies: { name: string; kind: string | null }[]; +} +interface Meta { + workspace_root: string; + packages: MetaPkg[]; +} +interface Crate { + name: string; + dir: string; // relative to the workspace root +} + +function readOr(path: string | undefined, fallback = ""): string { + if (!path) return fallback; + try { + return readFileSync(path, "utf8"); + } catch { + return fallback; + } +} + +// Cargo.lock is TOML; parse it as such. A dependency entry is "name" or +// "name version (source)" -- keep the leading name. source/checksum/dependencies +// are absent for path/workspace members, hence the defaults. +function parseLock(text: string): LockPkg[] { + const doc = parseToml(text) as { package?: Record[] }; + return (doc.package ?? []).map((p) => ({ + name: String(p.name), + version: String(p.version ?? ""), + source: String(p.source ?? ""), + checksum: String(p.checksum ?? ""), + deps: ((p.dependencies as string[]) ?? []).map((d) => d.split(" ")[0]), + })); +} + +// name -> canonical fingerprint of all its locked entries (a name may resolve +// to several versions). deps stay in lock order inside an entry; the entries +// themselves are sorted so the fingerprint is order-independent. +function lockFingerprint(pkgs: LockPkg[]): Map { + const byName = new Map(); + for (const p of pkgs) { + const entry = JSON.stringify([p.version, p.source, p.checksum, p.deps]); + (byName.get(p.name) ?? byName.set(p.name, []).get(p.name)!).push(entry); + } + const fp = new Map(); + for (const [name, entries] of byName) fp.set(name, JSON.stringify(entries.sort())); + return fp; +} + +// Names whose locked fingerprint differs between two lock files. +function lockChanged(oldPkgs: LockPkg[], newPkgs: LockPkg[]): string[] { + const o = lockFingerprint(oldPkgs); + const n = lockFingerprint(newPkgs); + const names = new Set([...o.keys(), ...n.keys()]); + return [...names].filter((name) => o.get(name) !== n.get(name)); +} + +// Workspace members reverse-reachable from `seeds` in the lock graph. +function reverseReachMembers(lock: LockPkg[], seeds: string[], members: string[]): string[] { + const radj = new Map(); // dep -> [dependents] + for (const p of lock) + for (const d of p.deps) (radj.get(d) ?? radj.set(d, []).get(d)!).push(p.name); + const cur = new Set(seeds); + let grew = seeds.length > 0; + while (grew) { + grew = false; + for (const x of [...cur]) + for (const r of radj.get(x) ?? []) if (!cur.has(r)) cur.add(r), (grew = true); + } + const memberSet = new Set(members); + return [...cur].filter((m) => memberSet.has(m)).sort(); +} + +// Grow `seeds` with every member that transitively depends on one, walking +// `deps` (name -> [workspace dep names]) in reverse to a fixed point. +function reverseClosure(deps: Map, names: string[], seeds: string[]): string[] { + const cur = new Set(seeds); + let grew = true; + while (grew) { + grew = false; + for (const n of names) { + if (cur.has(n)) continue; + if ((deps.get(n) ?? []).some((d) => cur.has(d))) cur.add(n), (grew = true); + } + } + return [...cur].sort(); +} + +// The crate whose directory contains `file`; longest prefix wins, so nested +// crates (e.g. pallets/x/mock) beat their parent. Null if unowned. +function owningCrate(file: string, crates: Crate[]): string | null { + let best: Crate | null = null; + for (const c of crates) + if (file.startsWith(c.dir + "/") && (!best || c.dir.length > best.dir.length)) best = c; + return best?.name ?? null; +} + +// Member names whose resolution changed between the base and head lock files, +// plus dep-name seeds from a changed root manifest. Returns null when Cargo.lock +// changed but there is no base lock to diff against: caller falls back to full. +function lockAffected( + changed: string[], + members: string[], + baseLock: string, + tomlDiff: string, +): string[] | null { + const headLock = parseLock(readFileSync("Cargo.lock", "utf8")); + const seeds = new Set(); + if (changed.includes("Cargo.lock")) { + if (baseLock.trim().length === 0) return null; + for (const c of lockChanged(parseLock(baseLock), headLock)) seeds.add(c); + } + if (changed.includes("Cargo.toml")) { + // Dep names from changed lines of the root manifest; tokens that are not + // package names simply match nothing in the lock graph. + for (const line of tomlDiff.split("\n")) { + const m = line.match(/^[+-]\s*([A-Za-z0-9_-]+)\s*=/); + if (m) seeds.add(m[1]); + } + } + return reverseReachMembers(headLock, [...seeds], members); +} + +function main(): void { + const [, , changedPath, baseLockPath, tomlDiffPath] = process.argv; + const changed = readOr(changedPath) + .split("\n") + .filter((s) => s.length > 0); + const baseLock = readOr(baseLockPath); + const tomlDiff = readOr(tomlDiffPath); + + const meta: Meta = JSON.parse( + execFileSync("cargo", ["metadata", "--no-deps", "--format-version", "1"], { + encoding: "utf8", + maxBuffer: MAX_BUFFER, + }), + ); + const root = meta.workspace_root; + const crates: Crate[] = meta.packages.map((p) => { + let dir = p.manifest_path.replace(/\/Cargo\.toml$/, ""); + if (dir.startsWith(root + "/")) dir = dir.slice(root.length + 1); + return { name: p.name, dir }; + }); + const names = crates.map((c) => c.name); + const nameSet = new Set(names); + // name -> [workspace deps], normal/build kinds only (dev edges can't reach a + // dependent through the no-dev-deps check; see header). + const deps = new Map(); + for (const p of meta.packages) + deps.set( + p.name, + p.dependencies + .filter((d) => (d.kind === null || d.kind === "build") && nameSet.has(d.name)) + .map((d) => d.name), + ); + + let extra: string[] = []; + if (changed.includes("Cargo.lock") || changed.includes("Cargo.toml")) { + const affected = lockAffected(changed, names, baseLock, tomlDiff); + if (affected === null) { + process.stdout.write(WORKSPACE_ARGS + "\n"); + return; + } + extra = affected; + } + + // Sort the changed files into owned / ignored / unattributable. + const files = changed.filter((f) => !HANDLED.some((re) => re.test(f))); + const touched = files + .map((f) => owningCrate(f, crates)) + .filter((x): x is string => x !== null); + const unattributable = files.filter( + (f) => owningCrate(f, crates) === null && !IGNORE.some((re) => re.test(f)), + ); + + let out: string; + if (changed.some((f) => GLOBAL.some((re) => re.test(f))) || unattributable.length > 0) { + out = WORKSPACE_ARGS; + } else { + const closure = reverseClosure(deps, names, [...new Set([...touched, ...extra])]).filter( + (n) => !EXCLUDED.includes(n), + ); + const allNonExcluded = names.filter((n) => !EXCLUDED.includes(n)).sort(); + if (closure.length === 0) out = ""; + else if ( + closure.length === allNonExcluded.length && + closure.every((c, i) => c === allNonExcluded[i]) + ) + out = WORKSPACE_ARGS; + else out = closure.map((p) => "-p " + p).join(" "); + } + process.stdout.write(out + "\n"); +} + +main(); diff --git a/scripts/package-lock.json b/scripts/package-lock.json new file mode 100644 index 000000000..a76a81767 --- /dev/null +++ b/scripts/package-lock.json @@ -0,0 +1,27 @@ +{ + "name": "midnight-node-scripts", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "midnight-node-scripts", + "version": "0.0.0", + "dependencies": { + "smol-toml": "^1.6.1" + } + }, + "node_modules/smol-toml": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", + "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + } + } +} diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 000000000..ad73d226f --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,10 @@ +{ + "name": "midnight-node-scripts", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Repo CI/dev scripts (TypeScript, run directly via node type-stripping).", + "dependencies": { + "smol-toml": "^1.6.1" + } +} From b554fba74f8836fc621f5daf8f7efcf963ed15d1 Mon Sep 17 00:00:00 2001 From: Giles Cope Date: Tue, 16 Jun 2026 11:22:00 +0100 Subject: [PATCH 4/4] feat: tweaks Signed-off-by: Giles Cope --- .github/workflows/feature-unification.yml | 18 ++++++++++--- Earthfile | 32 ++++++++--------------- scripts/feature-unification-scope.ts | 9 ++++--- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/.github/workflows/feature-unification.yml b/.github/workflows/feature-unification.yml index 710a94fdf..a2d64c1e3 100644 --- a/.github/workflows/feature-unification.yml +++ b/.github/workflows/feature-unification.yml @@ -56,10 +56,20 @@ jobs: # if: steps.guard.outputs.hit != 'true' # run: scripts/free-disk-space.sh - # Scope is computed inside the earthly target now (the - # +feature-unification-inputs LOCALLY target does the only git work, then - # scripts/feature-unification-scope.ts attributes the diff in-container). - # An empty scope skips the cargo-hack check from within the target. + # The only git-coupled work: git history exists on the host, and `--ci` + # (strict) forbids LOCALLY inside the build, so the scope inputs are + # produced here and COPYed into the container. HEAD^1 is the PR base on a + # merge-commit checkout (fetch-depth: 2 above). + - name: Collect scope inputs + if: steps.guard.outputs.hit != 'true' + run: | + mkdir -p .scope + git diff --name-only HEAD^1 HEAD > .scope/changed.txt + git show HEAD^1:Cargo.lock > .scope/base-lock.txt 2>/dev/null || : > .scope/base-lock.txt + git diff HEAD^1 HEAD -- Cargo.toml > .scope/toml-diff.txt 2>/dev/null || : > .scope/toml-diff.txt + + # scripts/feature-unification-scope.ts attributes the diff to crates + # in-container and skips the cargo-hack check when nothing is affected. - name: Run feature unification check if: steps.guard.outputs.hit != 'true' run: | diff --git a/Earthfile b/Earthfile index 34f852c3f..55ea87dba 100644 --- a/Earthfile +++ b/Earthfile @@ -854,30 +854,21 @@ check-rust: ENV SKIP_WASM_BUILD=1 -# feature-unification-inputs is the entire host footprint of the scope step: -# three text files the scoper reads, all derived from git history (which only -# exists on the host, not in a container). HEAD^1 is the PR base on a -# merge-commit checkout; override with --SCOPE_BASE_REF for local runs. -feature-unification-inputs: - LOCALLY - ARG SCOPE_BASE_REF=HEAD^1 - RUN mkdir -p .scope && \ - git diff --name-only "$SCOPE_BASE_REF" HEAD > .scope/changed.txt && \ - { git show "$SCOPE_BASE_REF:Cargo.lock" > .scope/base-lock.txt 2>/dev/null || : > .scope/base-lock.txt; } && \ - { git diff "$SCOPE_BASE_REF" HEAD -- Cargo.toml > .scope/toml-diff.txt 2>/dev/null || : > .scope/toml-diff.txt; } - SAVE ARTIFACT .scope/changed.txt changed.txt - SAVE ARTIFACT .scope/base-lock.txt base-lock.txt - SAVE ARTIFACT .scope/toml-diff.txt toml-diff.txt - # check-feature-unification verifies each crate compiles without dev-deps, # catching issues where workspace feature unification masks missing dependencies. # The partner-chains demo crates are excluded: they are upstream examples, not # shipped artifacts, and cost ~5min of the serial check. # # Scope is computed in-container by scripts/feature-unification-scope.ts (the -# reverse-dependency closure of the crates the PR diff touches) so the only -# host work is the git reads in +feature-unification-inputs. An empty scope -# (nothing compile-relevant changed) skips the check entirely. +# reverse-dependency closure of the crates the PR diff touches). It reads three +# git-derived files from .scope/, which must exist before the build -- git +# history only lives on the host, and `--ci` (strict) forbids LOCALLY. The CI +# workflow writes them; for a local run, from the repo root: +# mkdir -p .scope +# git diff --name-only HEAD^1 HEAD > .scope/changed.txt +# git show HEAD^1:Cargo.lock > .scope/base-lock.txt +# git diff HEAD^1 HEAD -- Cargo.toml > .scope/toml-diff.txt +# An empty scope (nothing compile-relevant changed) skips the check entirely. check-feature-unification: FROM +check-rust-prepare CACHE --sharing shared --id cargo-git /usr/local/cargo/git @@ -891,9 +882,8 @@ check-feature-unification: ledger LICENSE node pallets primitives README.md res runtime \ metadata rustfmt.toml util tests relay partner-chains COMPACTC_VERSION . COPY scripts/feature-unification-scope.ts scripts/feature-unification-scope.ts - COPY +feature-unification-inputs/changed.txt \ - +feature-unification-inputs/base-lock.txt \ - +feature-unification-inputs/toml-diff.txt .scope/ + # git-derived scope inputs, produced on the host before the build (see above) + COPY .scope/changed.txt .scope/base-lock.txt .scope/toml-diff.txt .scope/ ENV SKIP_WASM_BUILD=1 ENV CARGO_INCREMENTAL=0 diff --git a/scripts/feature-unification-scope.ts b/scripts/feature-unification-scope.ts index 7c81e6f0e..39d41b45f 100644 --- a/scripts/feature-unification-scope.ts +++ b/scripts/feature-unification-scope.ts @@ -8,9 +8,9 @@ // file: the base commit's Cargo.lock (empty if none) // file: `git diff` of the root Cargo.toml (empty if untouched) // -// All git access lives in the Earthfile's `feature-unification-inputs` LOCALLY -// target, which produces these three files; this script is pure computation -// over them plus `cargo metadata` and the head `Cargo.lock` (both read from the +// All git access happens before the build (the CI workflow, or a few git +// commands locally) and lands in .scope/; this script is pure computation over +// those files plus `cargo metadata` and the head `Cargo.lock` (read from the // current workspace, i.e. inside the check container). Prints the package // selection args for `cargo hack check --no-dev-deps`: // @@ -61,7 +61,10 @@ const IGNORE = [ /\.md$/, /^LICENSE/, /^Earthfile$/, + /^\.gitignore$/, + // the scoper's own files: changing them can't change whether crates compile /^scripts\/feature-unification-scope\.ts$/, + /^scripts\/package(-lock)?\.json$/, ]; interface LockPkg {