cli-tests.yaml

name: cli-tests
# Runs the Devbox CLI tests

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
  push:
    branches:
      - main
  merge_group:
    branches:
      - main
  workflow_call:
    inputs:
      run-mac-tests:
        type: boolean
  workflow_dispatch:
    inputs:
      run-mac-tests:
        type: boolean
        description: Run tests on macOS
      example-debug:
        type: boolean
        description: Run example tests with DEVBOX_DEBUG=1 to increase verbosity
  schedule:
    - cron: '30 8 * * *' # Run nightly at 8:30 UTC

permissions:
  contents: read
  pull-requests: read

defaults:
  run:
    # Explicitly setting the shell to bash runs commands with
    # `bash --noprofile --norc -eo pipefail` instead of `bash -e`.
    shell: bash

env:
  DEVBOX_DEBUG: 1
  DEVBOX_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  HOMEBREW_NO_ANALYTICS: 1
  HOMEBREW_NO_AUTO_UPDATE: 1
  HOMEBREW_NO_EMOJI: 1
  HOMEBREW_NO_ENV_HINTS: 1
  HOMEBREW_NO_INSTALL_CLEANUP: 1
  NIX_CONFIG: |
    access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}

jobs:
  build-devbox:
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-go@v6
        with:
          go-version-file: ./go.mod
      - name: Build devbox
        run: go build -o dist/devbox ./cmd/devbox
      - name: Upload devbox artifact
        uses: actions/upload-artifact@v7
        with:
          name: devbox-${{ runner.os }}-${{ runner.arch }}
          path: ./dist/devbox
          retention-days: 7

  typos:
    name: Spell Check with Typos
    if: github.ref != 'refs/heads/main'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - uses: crate-ci/typos@v1.16.26

  flake-test:
    name: Test Flake Build
    if: github.ref != 'refs/heads/main'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - name: Install devbox
        uses: jetify-com/devbox-install-action@jl/migrate-installer
        with:
          enable-cache: true
      - name: Build flake
        run: |
          if ! devbox run build-flake; then
            echo "::warning::If this fails, you probably have to run 'devbox run update-hash'"
            exit 1
          fi
      - run: ./result/bin/devbox version

  golangci-lint:
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v7

      - name: Install devbox
        uses: jetify-com/devbox-install-action@jl/migrate-installer
        with:
          enable-cache: true

      - name: Mount golang cache
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/golangci-lint
            ~/.cache/go-build
            ~/go/pkg
          key: go-${{ runner.os }}-${{ hashFiles('go.sum') }}

      - run:  devbox run lint

  test:
    needs: build-devbox
    strategy:
      # One failing shard shouldn't cancel the rest. This matrix gates releases
      # via `cli-release`, so when it goes red we need the full picture of what
      # broke, not just the first shard to fail.
      fail-fast: false
      matrix:
        is-main:
          - ${{ github.ref == 'refs/heads/main' && 'is-main' || 'not-main' }}
        os: [ubuntu-latest, macos-latest]
        # This is an optimization that runs tests twice, with and without
        # the devbox.json tests. We can require the other tests to complete before
        # merging, while keeping the others as an additional non-required signal
        run-project-tests: ["project-tests-only", "project-tests-off"]
        # The testscripts are bound by per-runner nix work (downloading package
        # closures and evaluating flakes), which barely parallelizes within a
        # single runner. We instead shard the scripts across multiple runners.
        # The runner count here must match DEVBOX_TEST_SHARD_TOTAL below. The
        # `test-result` job aggregates all shards into one required status check.
        shard: [0, 1, 2]
        # NOTE: this job used to run a `nix-version` matrix, but the
        # devbox-install-action we now use to install Nix doesn't let us pin a
        # specific Nix version. The matrix legs were therefore running identical
        # duplicates (3x the jobs, 3x the flake surface, no extra coverage), so
        # it was removed. Cross-version coverage is currently provided by the
        # separate `test-nix-versions` job. Once the install action supports
        # specifying a Nix version, re-add the matrix here and wire it into the
        # "Install devbox" step.
        exclude:
          # Only runs tests on macos if explicitly requested, or on a schedule
          - os: "${{ (inputs.run-mac-tests || github.event.schedule != '') && 'dummy' || 'macos-latest' }}"


    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    env:
      # For devbox.json tests, we default to non-debug mode since the debug output is less useful than for unit testscripts.
      # But we allow overriding via inputs.example-debug
      DEVBOX_DEBUG: ${{ (matrix.run-project-tests == 'project-tests-off' || inputs.example-debug) && '1' || '0' }}
      DEVBOX_GOLANG_TEST_TIMEOUT: "${{ (github.ref == 'refs/heads/main' || inputs.run-mac-tests) && '1h' || '30m' }}"
      # Shard the testscripts across runners. DEVBOX_TEST_SHARD_TOTAL must match
      # the length of the `shard` matrix axis above.
      DEVBOX_TEST_SHARD_INDEX: ${{ matrix.shard }}
      DEVBOX_TEST_SHARD_TOTAL: 3
    steps:
      - name: clear directories to reduce disk usage
        # https://github.com/actions/runner-images/issues/2840#issuecomment-1284059930
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - uses: actions/checkout@v7
      - name: Mount golang cache
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg
          key: go-devbox-tests-${{ runner.os }}-${{ hashFiles('go.sum') }}
      - name: Install additional shells (dash, zsh)
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
            sudo apt-get update
            sudo apt-get install dash zsh
          elif [ "$RUNNER_OS" == "macOS" ]; then
            brew update
            brew install dash zsh
          fi
      - name: Install devbox
        uses: jetify-com/devbox-install-action@jl/migrate-installer
        with:
          enable-cache: true
      - name: Setup Nix GitHub authentication
        run: |
          # Setup github authentication to ensure GitHub's API rate limits are not
          # hit when Nix resolves `github:` flake refs (e.g. fetching the latest
          # commit of nixpkgs-unstable). Anonymous requests are limited to 60/hr,
          # which is quickly exhausted by the parallel test suite and surfaces as
          # "unable to download '.../commits/...': HTTP error 403".
          #
          # `access-tokens` set via NIX_CONFIG or the user's ~/.config/nix/nix.conf
          # is only honored when the invoking user is a trusted user. The Nix daemon
          # (which can run as a different user) does not read the user config at all.
          # The system-wide /etc/nix/nix.conf is always trusted and read by both the
          # client and the daemon, so write the token there on every platform.
          echo "Configuring system-wide Nix config (/etc/nix/nix.conf)"
          sudo mkdir -p /etc/nix
          if [ ! -f /etc/nix/nix.conf ]; then
            echo "# Nix configuration" | sudo tee /etc/nix/nix.conf > /dev/null
          fi
          echo "access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}" | sudo tee -a /etc/nix/nix.conf > /dev/null

          # Restart the Nix daemon so it picks up the new configuration.
          if [ "$RUNNER_OS" == "macOS" ]; then
            sudo launchctl stop org.nixos.nix-daemon || true
            sudo launchctl start org.nixos.nix-daemon || true
            sleep 2  # Give the daemon time to restart
          elif systemctl list-unit-files nix-daemon.service >/dev/null 2>&1; then
            # Only restart when the unit exists, and let a real restart failure
            # fail the job rather than silently keeping the old config.
            sudo systemctl restart nix-daemon
          fi

          # Also configure the user config as a backup for trusted-user setups.
          mkdir -p ~/.config/nix
          echo "access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}" > ~/.config/nix/nix.conf
      - name: Run fast tests
        if: matrix.run-project-tests == 'project-tests-off'
        run: |
          echo "::group::Nix version"
          nix --version
          echo "::endgroup::"
          echo "::group::Contents of /etc/nix/nix.conf"
          cat /etc/nix/nix.conf || true
          echo "::endgroup::"
          echo "::group::Resolved Nix config"
          nix show-config --extra-experimental-features nix-command
          echo "::endgroup::"
          devbox run go test -v -timeout $DEVBOX_GOLANG_TEST_TIMEOUT ./...
      - name: Run project (slow) tests
        if:  matrix.run-project-tests == 'project-tests-only'
        run: devbox run test-projects-only

  # Aggregates the sharded `test` matrix into a single status check. Because the
  # shard matrix produces per-shard job names (e.g.
  # "test (not-main, ubuntu-latest, project-tests-off, 0)"), branch protection
  # should require this stable `test-result` check instead of the individual
  # matrix jobs.
  test-result:
    name: test-result
    if: always()
    needs: test
    runs-on: ubuntu-latest
    steps:
      - name: Check that all test shards passed
        run: |
          result="${{ needs.test.result }}"
          echo "Aggregate result of the test matrix: $result"
          if [ "$result" != "success" ]; then
            echo "::error::One or more test shards did not succeed ($result)."
            exit 1
          fi

  auto-nix-install: # ensure Devbox installs nix and works properly after installation.
    needs: build-devbox
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        use-detsys: [true, false]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v7
      - name: Download devbox
        uses: actions/download-artifact@v8
        with:
          name: devbox-${{ runner.os }}-${{ runner.arch }}
      - name: Add devbox to path
        run: |
          chmod +x ./devbox
          sudo mv ./devbox /usr/local/bin/
      - name: Install nix and devbox packages
        run: |
          export NIX_INSTALLER_NO_CHANNEL_ADD=1
          export DEVBOX_FEATURE_DETSYS_INSTALLER=${{ matrix.use-detsys }}

          # Setup github authentication BEFORE running devbox to ensure Github's rate limits are not hit.
          # Configure user config first (Nix installer will respect this)
          mkdir -p ~/.config/nix
          echo "access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}" > ~/.config/nix/nix.conf

          # Run devbox which will auto-install Nix if needed
          devbox run echo "Installing packages..."

          # After Nix is installed, configure system-wide config for the daemon on macOS
          if [ "$RUNNER_OS" == "macOS" ]; then
            echo "Configuring system-wide Nix config for macOS daemon"
            # Check if file exists, create directory if needed
            if [ ! -f /etc/nix/nix.conf ]; then
              sudo mkdir -p /etc/nix
              echo "# Nix configuration" | sudo tee /etc/nix/nix.conf > /dev/null
            fi
            echo "access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}" | sudo tee -a /etc/nix/nix.conf
            # Restart nix daemon to pick up the new configuration
            sudo launchctl stop org.nixos.nix-daemon || true
            sudo launchctl start org.nixos.nix-daemon || true
            sleep 2  # Give daemon time to restart
          fi
      - name: Test removing package
        run: devbox rm go

  # Run a sanity test to make sure Devbox can install and remove packages with
  # the last few Nix releases.
  test-nix-versions:
    needs: build-devbox
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        nix-version: [2.18.0, 2.19.2, 2.24.7, 2.30.2, 2.35.2]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v7
      - name: Download devbox
        uses: actions/download-artifact@v8
        with:
          name: devbox-${{ runner.os }}-${{ runner.arch }}
      - name: Add devbox to path
        run: |
          chmod +x ./devbox
          sudo mv ./devbox /usr/local/bin/
      - name: Install Nix
        uses: DeterminateSystems/nix-installer-action@v4
        with:
          logger: pretty
          # access-tokens is written to the system-wide /etc/nix/nix.conf so both
          # the Nix client and daemon authenticate to GitHub and avoid the
          # anonymous API rate limit (HTTP 403) when resolving `github:` flake refs.
          extra-conf: |
            experimental-features = ca-derivations fetch-closure
            access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
          nix-package-url: https://releases.nixos.org/nix/nix-${{ matrix.nix-version }}/nix-${{ matrix.nix-version }}-${{ runner.arch == 'X64' && 'x86_64' || 'aarch64' }}-${{ runner.os == 'macOS' && 'darwin' || 'linux' }}.tar.xz
      - name: Run devbox install, devbox run, devbox rm
        run: |
          echo "::group::Nix version"
          nix --version
          echo "::endgroup::"
          echo "::group::Contents of /etc/nix/nix.conf"
          cat /etc/nix/nix.conf || true
          echo "::endgroup::"
          echo "::group::Resolved Nix config"
          nix show-config --extra-experimental-features nix-command
          echo "::endgroup::"
          devbox install
          devbox run -- echo "Hello from devbox!"
          devbox rm go