Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion cmd/nova-fleet/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,11 @@ passes the stop to its listener as SIGINT and waits for it (a running job is can
`KillMode=mixed` kills whatever is left in the unit once `run.sh` exits, or after `TimeoutStopSec` (5 minutes). On macOS
launchd kills what is left of the job's process group.
A Linux unit sets `LimitMEMLOCK=infinity`: the kernel charges locked memory (io_uring rings) to the login across all its
runners, and systemd's 8 MiB default refuses that. It also sets `TMPDIR`, by default `<home>/.cache/runner-tmp`
runners, and systemd's 8 MiB default refuses that. On a host whose kernel isolates CPUs from the scheduler
(`isolcpus`, listed in `/sys/devices/system/cpu/isolated`) the unit sets `CPUAffinity=` to the online CPUs less the
isolated ones (`github_runner_cpus`), so neither the runner nor anything its jobs start lands on an isolated CPU, where
the scheduler never balances it and every thread placed there shares one core; systemd otherwise starts every unit on
all CPUs. A host that isolates none gets no affinity line. It also sets `TMPDIR`, by default `<home>/.cache/runner-tmp`
(`github_runner_tmpdir`), one directory for all the login's runners that the play makes (the login's, `0700`) before
the units start: a job's temporary files land there and not in `/tmp`, and the health play's disk clean reads the
directory from the units. A macOS daemon gets no `TMPDIR` unless `github_runner_env` names one, so its jobs use `/tmp`,
Expand Down
21 changes: 21 additions & 0 deletions cmd/nova-fleet/github_runner_path_functional_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,27 @@ func TestRunnerEnvPathStillWins(t *testing.T) {
assert.Equal(t, 1, strings.Count(plist, "<key>PATH</key>"))
}

// The Linux unit names the online CPUs less the kernel's isolated ones (a host booted with isolcpus=1-15), and no
// affinity where none is isolated, so every other host's unit is unchanged.
func TestRunnerServiceKeepsOffIsolatedCPUs(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("ansible-playbook"); err != nil {
t.Skip("ansible-core is not installed on this machine; TestRunnerUnitNamesTheBalancedCPUs still runs")
}
service, plist, _ := renderRunnerUnits(t, "-e", "github_runner_cpus_online=0-31", "-e", "github_runner_cpus_isolated=1-15")
assert.Contains(t, service, "\nCPUAffinity=0 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31\n")
assert.Equal(t, 1, strings.Count(service, "CPUAffinity="))
assert.NotContains(t, plist, "CPUAffinity")

service, _, _ = renderRunnerUnits(t, "-e", "github_runner_cpus_online=0-7,16-23", "-e", "github_runner_cpus_isolated=2,4-7,16")
assert.Contains(t, service, "\nCPUAffinity=0 1 3 17 18 19 20 21 22 23\n")

service, _, _ = renderRunnerUnits(t, "-e", "github_runner_cpus_online=0-31", "-e", "github_runner_cpus_isolated=")
assert.NotContains(t, service, "CPUAffinity", "a host that isolates no CPU keeps the scheduler's default")
service, _, _ = renderRunnerUnits(t)
assert.NotContains(t, service, "CPUAffinity", "nothing read: no affinity")
}

// The Linux unit's TMPDIR is the login's runner-tmp unless the environment names one; the environment is the machine
// record's (the inventory's nova_runner_env) when no -e replaces it. The plist sets none by default.
func TestRunnerTmpdirAndTheRecordsEnvironmentRender(t *testing.T) {
Expand Down
28 changes: 28 additions & 0 deletions cmd/nova-fleet/github_runner_path_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,34 @@ func TestRunnerServiceLiftsTheLockedMemoryLimit(t *testing.T) {
assert.NotContains(t, s, "sudo")
}

// A Linux runner's unit names the CPUs the scheduler balances, read from the host's online and isolated lists before any
// unit is written (in --check too), and only when the kernel isolates some: a job on an isolated CPU is never balanced
// off it. The render is TestRunnerServiceKeepsOffIsolatedCPUs.
func TestRunnerUnitNamesTheBalancedCPUs(t *testing.T) {
t.Parallel()
b, err := os.ReadFile("roles/github-runner/templates/runner.service.j2")
require.NoError(t, err)
s := string(b)
service := s[strings.Index(s, "\n[Service]\n"):strings.Index(s, "\n[Install]\n")]
assert.Contains(t, service, "{% if github_runner_cpus | string | length > 0 %}\n")
assert.Contains(t, service, "\nCPUAffinity={{ github_runner_cpus }}\n")
assert.Equal(t, 1, strings.Count(s, "CPUAffinity="))

d := runnerDefaults(t)
cpus := asText(d["github_runner_cpus"])
assert.Contains(t, cpus, "github_runner_cpus_isolated")
assert.Contains(t, cpus, "github_runner_cpus_online")

b, err = os.ReadFile("roles/github-runner/tasks/main.yml")
require.NoError(t, err)
m := string(b)
read := strings.Index(m, `src: "/sys/devices/system/cpu/{{ item }}"`)
require.GreaterOrEqual(t, read, 0, "main.yml reads the CPU lists")
assert.Contains(t, m, "loop: [online, isolated]")
assert.Less(t, read, strings.Index(m, "include_tasks: record.yml"), "read before any unit is written")
assert.Less(t, read, strings.Index(m, "include_tasks: scope.yml"), "read before any unit is written")
}

// Both service files set PATH once: github_runner_env's PATH if it has one, else the default above.
func TestRunnerServiceFilesSetPath(t *testing.T) {
t.Parallel()
Expand Down
16 changes: 16 additions & 0 deletions cmd/nova-fleet/roles/github-runner/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,22 @@ github_runner_bin_dir: "{{ nova_bin_dir if github_runner_user == ansible_facts['
github_runner_default_path_linux: "{{ github_runner_bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin"
github_runner_default_path_darwin: "{{ nova_bin_dir }}:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"

# The CPUs a Linux runner and everything its jobs start run on (the unit's CPUAffinity=), space-separated: the online
# CPUs less those the kernel isolates from the scheduler (isolcpus), read from /sys/devices/system/cpu/{online,isolated}
# by tasks/main.yml. A task on an isolated CPU is never balanced off it, so every thread a job places there shares that
# one core; systemd starts every unit on all CPUs, isolated or not. Empty where the kernel isolates none: the unit then
# names no affinity.
github_runner_cpus: >-
{%- set gone = [] -%}
{%- for r in (github_runner_cpus_isolated | default('') | string | trim).split(',') if r -%}
{%- for c in range(r.split('-')[0] | int, r.split('-')[-1] | int + 1) -%}{%- set _ = gone.append(c) -%}{%- endfor -%}
{%- endfor -%}
{%- set kept = [] -%}
{%- for r in (github_runner_cpus_online | default('') | string | trim).split(',') if r and gone -%}
{%- for c in range(r.split('-')[0] | int, r.split('-')[-1] | int + 1) if c not in gone -%}{%- set _ = kept.append(c) -%}{%- endfor -%}
{%- endfor -%}
{{ kept | join(' ') }}

# Linux only: run the runner's bin/installdependencies.sh (system packages) once per
# release on the host.
github_runner_install_dependencies: true
Expand Down
26 changes: 26 additions & 0 deletions cmd/nova-fleet/roles/github-runner/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,32 @@
tags: [deploy-keys]
when: github_runner_deploy_keys | length > 0

# The CPUs each Linux unit names (github_runner_cpus): the online ones less the kernel's isolated ones. A kernel without
# the isolated file isolates none. Read in --check too, so a plan shows the affinity it would write.
- name: the online and isolated CPUs (Linux)
ansible.builtin.slurp: { src: "/sys/devices/system/cpu/{{ item }}" }
loop: [online, isolated]
register: github_runner_cpu_files
# ignored: a missing isolated file is read as none (below); a missing online file leaves no CPU, refused below
failed_when: false
when: ansible_facts['system'] == 'Linux'

- name: the CPUs the runners may use (Linux)
ansible.builtin.set_fact:
github_runner_cpus_online: "{{ (github_runner_cpu_files.results[0].content | default('') | b64decode) | trim }}"
github_runner_cpus_isolated: "{{ (github_runner_cpu_files.results[1].content | default('') | b64decode) | trim }}"
when: ansible_facts['system'] == 'Linux'

- name: some CPU is left to the runners (Linux)
ansible.builtin.assert:
quiet: true
that: github_runner_cpus_isolated == '' or github_runner_cpus | string | length > 0
fail_msg: >-
github-runner: {{ inventory_hostname }} isolates CPUs {{ github_runner_cpus_isolated }} of its online
{{ github_runner_cpus_online or 'none' }}, which leaves its runners no CPU the scheduler balances; run: cat
/proc/cmdline (on {{ inventory_hostname }}), then leave CPUs outside isolcpus
when: ansible_facts['system'] == 'Linux'

- name: one url, named with -e github_runner_url
ansible.builtin.include_tasks: scope.yml
when: github_runner_url is defined
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ Environment=RUNNER_MANUALLY_TRAP_SIG=1
LimitMEMLOCK=infinity
CPUWeight=10000
IOWeight=10000
{% if github_runner_cpus | string | length > 0 %}
# the CPUs the scheduler balances (github_runner_cpus): never one the kernel isolates (isolcpus), where every thread of
# a job would share one core
CPUAffinity={{ github_runner_cpus }}
{% endif %}
Nice=-10
ExecStart={{ github_runner_dir }}/run.sh
Restart=always
Expand Down
Loading