Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/nova-fleet/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1703,7 +1703,7 @@ version bump installs the new directory beside the old and moves the links.
| Variable | Default | |
|---|---|---|
| `nova_sdk_dir` | `<home>/sdk` | the SDK directory, used by its real path |
| `nova_sdk_shared` | on for a Linux runner whose runners are another login's (`github_runner_user`) | Linux: the SDK in `/opt/nova/sdk` (`nova_shared_dir`, made by `github-runners` with a runner login of its own), the login's and readable by every login, with `nova_sdk_dir` a link to it, so a runner login of its own runs the same tools. The shared SDK is installed fresh (its files name their own path) while `~/sdk` stays in use; at the end of the run `~/sdk` becomes a link to it (`SDK-TOOLCHAIN SHARE`) and the old one is deleted. Nothing in it is left writable by other logins, and a run fails on anything in it they cannot read; an old `~/sdk` holding what the shared one lacks (an older version a link names, a directory put there by hand) stays in use and the run fails naming it (`SDK-TOOLCHAIN SHARE-HELD`) |
| `nova_sdk_shared` | on for a Linux runner whose runners are another login's (`github_runner_user`) | Linux: the SDK in `/opt/nova/sdk` (`nova_shared_dir`, made by `github-runners` with a runner login of its own), the login's and readable by every login, with `nova_sdk_dir` a link to it, so a runner login of its own runs the same tools. The shared SDK is installed fresh (its files name their own path) while `~/sdk` stays in use; at the end of the run `~/sdk` becomes a link to it (`SDK-TOOLCHAIN SHARE`) and the old one is deleted. Nothing in it is left writable by other logins, and a run fails on anything in it they cannot read; an old `~/sdk` holding what the shared one lacks (an older version a link names, a directory put there by hand) stays in use and the run fails naming it (`SDK-TOOLCHAIN SHARE-HELD`; compared two levels down, and further into rustup's toolchains and cargo tools and each opam switch's packages and commands); and while any process runs from the old `~/sdk` (its executable, working directory, an open or mapped file) nothing is moved or deleted and the run fails naming the processes, which it never stops (`SDK-TOOLCHAIN SDK-IN-USE`) |
| `nova_sdk_tools` | the toolchain above | one entry per tool: `name`, `version`, `dir`, `probe`, `links`, `kind`, `build`, `alias`, `wrapper`, `run`, and a `linux` and/or `darwin` pin (a pin's own `build` replaces the tool's on that platform); a list given with `-e` replaces it whole |
| `nova_sdk_dotnet_more` | the .NET 8.0 SDK | more SDKs in the dotnet directory, each a `version` and a `url` and `sha512` per platform |
| `nova_sdk_rustup` | rustup 1.29.1; `1.98.1` with `clippy` and `rustfmt`, and `nightly-2026-10-01` | `version`, the `linux` rustup-init `url`, `sha256` and `host`, the `toolchains` (each `name`, `components`, `targets`; the first is on PATH) and the commands `links` puts in `~/sdk/bin` |
Expand Down
23 changes: 23 additions & 0 deletions cmd/nova-fleet/github_runner_shared_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,29 @@ func TestSDKToolchainSharesTheSDKAsTheLogin(t *testing.T) {
assert.Less(t, taskIndex(link, "everything in the shared SDK is readable by other logins"), taskIndex(link, "the login's own SDK is moved aside and nova_sdk_dir names the shared SDK"))
assert.Less(t, taskIndex(link, "an old SDK holding what the shared one lacks is kept"), taskIndex(link, "the login's own SDK is moved aside and nova_sdk_dir names the shared SDK"))
assert.Less(t, taskIndex(link, "the login's own SDK is moved aside and nova_sdk_dir names the shared SDK"), taskIndex(link, "the SDK moved aside is deleted"))
// compared below the top: a toolchain, a cargo tool or an opam package only the old SDK holds is held
extra := asText(runnerTask(t, link, "what the login's own SDK holds that the shared one lacks")["ansible.builtin.shell"])
assert.Contains(t, extra, `for e in * .[!.]* */* */.[!.]* rustup-*/cargo/bin rustup-*/toolchains/* rustup-*/cargo/bin/* opam/*/lib opam/*/bin opam/*/lib/* opam/*/bin/*; do`)
assert.Contains(t, extra, `if [ "$up" = . ]; then echo "$e"; fi`, "each path once, at its highest missing level")
// nothing is moved or deleted while a process runs from it, and nothing is stopped
for _, step := range [][2]string{
{"the login's own SDK is in use by no process", "the login's own SDK is moved aside and nova_sdk_dir names the shared SDK"},
{"the SDK moved aside is in use by no process", "the SDK moved aside is deleted"},
} {
check := runnerTask(t, link, step[0])
assert.Equal(t, "in-use.yml", asText(check["ansible.builtin.include_tasks"]))
assert.Equal(t, taskIndex(link, step[0])+1, taskIndex(link, step[1]), "the check comes right before %q", step[1])
}
inUse := taskFileOf(t, sdkToolchainRole+"/tasks/in-use.yml")
for _, task := range inUse {
assert.NotContains(t, task, "become", "task %q: the SDK is the login's", asText(task["name"]))
}
scan := asText(runnerTask(t, inUse, "no process runs from {{ sdk_in_use_dir }}")["ansible.builtin.shell"])
for _, want := range []string{`for f in "$p/exe" "$p/cwd" "$p"/fd/*; do`, `grep -q -F " $d/" "$p/maps"`} {
assert.Contains(t, scan, want)
}
assert.NotContains(t, scan, "kill")
assert.Contains(t, asText(runnerTask(t, inUse, "a directory in use is neither moved nor deleted")["ansible.builtin.fail"].(map[string]any)["msg"]), "SDK-TOOLCHAIN SDK-IN-USE")

b, err := os.ReadFile(sdkToolchainRole + "/defaults/main.yml")
require.NoError(t, err)
Expand Down
34 changes: 34 additions & 0 deletions cmd/nova-fleet/roles/sdk-toolchain/tasks/in-use.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
---
# Nothing is moved or deleted out from under a running process: a process whose executable, working directory, open
# file or mapped file is in sdk_in_use_dir (the login's own SDK before it is moved aside, the moved-aside one before it
# is deleted) fails the run naming it, and is never stopped. A JVM or a TLC run reads its runtime's files long after it
# starts, so a file deleted under it fails it later. The login reads its own processes' /proc entries; another login's
# cannot reach this directory (the login's home is closed to it).
- name: no process runs from {{ sdk_in_use_dir }}
ansible.builtin.shell: |
set -eu
d={{ sdk_in_use_dir | quote }}
[ -d "$d" ] && [ ! -L "$d" ] || exit 0
# the kernel names resolved paths in /proc, so the directory is compared by its own (a home under a linked /home)
d=$(cd "$d" && pwd -P)
for p in /proc/[0-9]*; do
[ "${p#/proc/}" = "$$" ] && continue
hit=
for f in "$p/exe" "$p/cwd" "$p"/fd/*; do
t=$(readlink "$f" 2>/dev/null) || continue
case "$t" in "$d" | "$d"/*) hit=1; break ;; esac
done
if [ -z "$hit" ] && grep -q -F " $d/" "$p/maps" 2>/dev/null; then hit=1; fi
if [ -n "$hit" ]; then echo "${p#/proc/}:$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null | cut -c1-60)"; fi
done
args: { executable: /bin/sh }
register: sdk_in_use
changed_when: false
check_mode: false

- name: a directory in use is neither moved nor deleted
ansible.builtin.fail:
msg: >-
SDK-TOOLCHAIN SDK-IN-USE host={{ inventory_hostname }} dir={{ sdk_in_use_dir }}: {{ sdk_in_use.stdout_lines[:20] | join('; ') }};
nothing is moved or deleted while they run. Run again once they have ended.
when: sdk_in_use.stdout_lines | length > 0
31 changes: 28 additions & 3 deletions cmd/nova-fleet/roles/sdk-toolchain/tasks/link.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
# the login later runs) and everything in it must be readable by them. Then nova_sdk_dir becomes a link to it: the
# login's own SDK is moved aside to <nova_sdk_dir>.home and the link made in one command, and the old SDK is deleted
# (Go's module cache in it is read-only, so it is made writable first). An old SDK holding anything the shared one lacks
# (an older version a link still names, a directory put there by hand) is left in use and the run fails naming it.
# (an older version a link still names, a directory put there by hand) is left in use and the run fails naming it, and
# so is one a process still runs from (in-use.yml, before the move and before the delete).
- name: nothing in the shared SDK is writable by other logins
ansible.builtin.shell: |
set -eu
Expand All @@ -30,13 +31,29 @@
ansible.builtin.stat: { path: "{{ nova_sdk_dir }}", follow: false, get_checksum: false }
register: sdk_dir_now

# Compared two levels down everywhere (a tool's directory and what is in it; dl/, the download cache the run fetches
# again by digest, aside) and one more where a managed tool keeps what a login adds to it: rustup's toolchains and
# cargo-installed tools, and each opam switch's packages and commands. Each path is named once, at its highest missing
# level.
- name: what the login's own SDK holds that the shared one lacks
ansible.builtin.shell: |
set -eu
shared={{ (nova_shared_dir ~ '/sdk') | quote }}
cd {{ nova_sdk_dir | quote }}
for e in * .[!.]*; do
for e in * .[!.]* */* */.[!.]* rustup-*/cargo/bin rustup-*/toolchains/* rustup-*/cargo/bin/* opam/*/lib opam/*/bin opam/*/lib/* opam/*/bin/*; do
[ -e "$e" ] || [ -L "$e" ] || continue
[ -e {{ (nova_shared_dir ~ '/sdk') | quote }}/"$e" ] || [ -L {{ (nova_shared_dir ~ '/sdk') | quote }}/"$e" ] || echo "$e"
case "$e" in dl | dl/*) continue ;; esac
# below a top-level link nothing is compared, the link itself is (a version it names that the shared SDK lacks is
# named on its own)
top=${e%%/*}
[ "$top" = "$e" ] || [ ! -L "$top" ] || continue
[ -e "$shared/$e" ] || [ -L "$shared/$e" ] && continue
up=$(dirname -- "$e")
while [ "$up" != . ]; do
[ -e "$shared/$up" ] || [ -L "$shared/$up" ] || break
up=$(dirname -- "$up")
done
if [ "$up" = . ]; then echo "$e"; fi
done
args: { executable: /bin/sh }
register: sdk_dir_extra
Expand All @@ -59,6 +76,10 @@
which {{ nova_shared_dir }}/sdk lacks; it stays in use. Move or delete them (or install them into the shared SDK) and run again.
when: sdk_dir_extra.stdout_lines | default([]) | length > 0

- name: the login's own SDK is in use by no process
ansible.builtin.include_tasks: in-use.yml
vars: { sdk_in_use_dir: "{{ nova_sdk_dir }}" }

- name: the login's own SDK is moved aside and nova_sdk_dir names the shared SDK
ansible.builtin.shell: |
set -eu
Expand All @@ -74,6 +95,10 @@
changed_when: "'SET' in sdk_link.stdout"
when: not ansible_check_mode

- name: the SDK moved aside is in use by no process
ansible.builtin.include_tasks: in-use.yml
vars: { sdk_in_use_dir: "{{ nova_sdk_dir ~ '.home' }}" }

- name: the SDK moved aside is deleted
ansible.builtin.shell: |
set -eu
Expand Down
122 changes: 122 additions & 0 deletions cmd/nova-fleet/sdk_toolchain_link_functional_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
//go:build functional

package main

import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// sdk_toolchain_link_functional_test.go runs the two scripts of sdk-toolchain's link.yml that decide whether the
// login's own SDK may go (github_runner_shared_test.go is the structural half): what it holds that the shared one lacks
// (SHARE-HELD), and whether a process runs from it (in-use.yml, SDK-IN-USE). Each runs under /bin/sh on temp trees, with
// the role's variables put in as the play quotes them.

func sdkToolchainScript(t *testing.T, file, task string, vars map[string]string) string {
t.Helper()
sh := asText(runnerTask(t, taskFileOf(t, sdkToolchainRole+"/tasks/"+file), task)["ansible.builtin.shell"])
for k, v := range vars {
sh = strings.ReplaceAll(sh, k, "'"+v+"'")
}
require.NotContains(t, sh, "{{", "every variable of the script is given")
return sh
}

func runScript(t *testing.T, sh string) []string {
t.Helper()
out, err := exec.Command("/bin/sh", "-c", sh).CombinedOutput()
require.NoError(t, err, "%s", out)
text := strings.TrimSpace(string(out))
if text == "" {
return nil
}
return strings.Split(text, "\n")
}

func makeTree(t *testing.T, root string, dirs, files []string, links map[string]string) {
t.Helper()
for _, d := range dirs {
require.NoError(t, os.MkdirAll(filepath.Join(root, d), 0o755))
}
for _, f := range files {
require.NoError(t, os.WriteFile(filepath.Join(root, f), nil, 0o644))
}
for name, target := range links {
require.NoError(t, os.Symlink(target, filepath.Join(root, name)))
}
}

func TestFunctionalSDKShareHeldNamesWhatOnlyTheOldSDKHolds(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
shared, own := filepath.Join(tmp, "shared"), filepath.Join(tmp, "own")
makeTree(t, shared,
[]string{"go-1.23/bin", "rustup-1/toolchains/1.80", "rustup-1/cargo/bin", "opam/default/lib/zarith", "opam/default/bin", "opam/sw2"},
[]string{"go-1.23/bin/go", "rustup-1/cargo/bin/cargo"},
map[string]string{"go": "go-1.23", "lost": "nowhere"})
makeTree(t, own,
[]string{"go-1.22/bin", "go-1.23/bin", "rustup-1/toolchains/1.80", "rustup-1/toolchains/stable/lib", "rustup-1/cargo/bin",
"opam/default/lib/zarith", "opam/default/lib/extra", "opam/default/bin", "opam/sw2/lib/pkg", "dl/old", "handmade/sub", "lost/deeper"},
[]string{"go-1.22/bin/go", "go-1.22/bin/gofmt", "go-1.23/bin/go", "go-1.23/vet", "rustup-1/cargo/bin/cargo",
"rustup-1/cargo/bin/cargo-nextest", "opam/default/bin/why3", ".hidden", "-old"},
map[string]string{"go": "go-1.22"})
sh := sdkToolchainScript(t, "link.yml", "what the login's own SDK holds that the shared one lacks", map[string]string{
"{{ (nova_shared_dir ~ '/sdk') | quote }}": shared,
"{{ nova_sdk_dir | quote }}": own,
})
got := runScript(t, sh)
// each once, at its highest missing level (a switch missing its lib directory too, a name starting with a dash);
// nothing of dl, even where the shared SDK has none; nothing again under the go link; a name the shared SDK holds
// only as a dangling link is climbed past, so what is below it is named
assert.ElementsMatch(t, []string{
"go-1.22", "go-1.23/vet", "handmade", ".hidden", "rustup-1/toolchains/stable",
"rustup-1/cargo/bin/cargo-nextest", "opam/default/lib/extra", "opam/default/bin/why3", "lost/deeper", "opam/sw2/lib", "-old",
}, got)

// an old SDK the shared one covers whole names nothing, and the script ends well with every glob unmatched
empty := filepath.Join(tmp, "empty")
require.NoError(t, os.Mkdir(empty, 0o755))
sh = sdkToolchainScript(t, "link.yml", "what the login's own SDK holds that the shared one lacks", map[string]string{
"{{ (nova_shared_dir ~ '/sdk') | quote }}": shared,
"{{ nova_sdk_dir | quote }}": empty,
})
assert.Empty(t, runScript(t, sh))
}

func TestFunctionalSDKInUseFindsAProcessRunningFromIt(t *testing.T) {
t.Parallel()
if runtime.GOOS != "linux" {
t.Skip("the scan reads /proc; the shared SDK is Linux only")
}
tmp := t.TempDir()
real := filepath.Join(tmp, "real")
require.NoError(t, os.MkdirAll(filepath.Join(real, "home", "sdk", "bin"), 0o755))
// the home is reached through a link, as under a linked /home: the kernel names the resolved path
require.NoError(t, os.Symlink(filepath.Join(real, "home"), filepath.Join(tmp, "home")))
dir := filepath.Join(tmp, "home", "sdk")
scan := func() []string {
return runScript(t, sdkToolchainScript(t, "in-use.yml", "no process runs from {{ sdk_in_use_dir }}",
map[string]string{"{{ sdk_in_use_dir | quote }}": dir}))
}
assert.Empty(t, scan(), "nothing runs from it yet")

cmd := exec.Command("sleep", "60")
cmd.Dir = filepath.Join(dir, "bin")
require.NoError(t, cmd.Start())
t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
got := scan()
require.Len(t, got, 1, "%v", got)
assert.True(t, strings.HasPrefix(got[0], strconv.Itoa(cmd.Process.Pid)+":sleep 60"), got[0])

// a link in place of the directory is not the SDK: nothing to find
assert.Empty(t, runScript(t, sdkToolchainScript(t, "in-use.yml", "no process runs from {{ sdk_in_use_dir }}",
map[string]string{"{{ sdk_in_use_dir | quote }}": filepath.Join(tmp, "home")})))
}
Loading