_scheduled-test-daily.yml

name: "> Scheduled Test (Daily)"

on:
  workflow_call:
    inputs:
      platform_environment:
        description: 'Environment to test against, that is staging or production'
        required: false
        default: 'staging'
        type: string
    secrets:
      AIGNOSTICS_CLIENT_ID_DEVICE_STAGING:
        required: true
      AIGNOSTICS_REFRESH_TOKEN_STAGING:
        required: true
      GCP_CREDENTIALS_STAGING:
        required: true
      BETTERSTACK_HEARTBEAT_URL_FLOWS_STAGING:
        required: true
      AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION:
        required: true
      AIGNOSTICS_REFRESH_TOKEN_PRODUCTION:
        required: true
      GCP_CREDENTIALS_PRODUCTION:
        required: true
      BETTERSTACK_HEARTBEAT_URL_FLOWS_PRODUCTION:
        required: true
      CODECOV_TOKEN:
        required: true
      SONAR_TOKEN:
        required: true
      SENTRY_DSN:
        required: true
  
env:
  # https://gist.github.com/NodeJSmith/e7e37f2d3f162456869f015f842bcf15
  PYTHONIOENCODING: "utf8"
  AIGNOSTICS_PLATFORM_ENVIRONMENT: ${{ inputs.platform_environment }}

jobs:

  test-scheduled-daily:
    runs-on: "ubuntu-latest"
    permissions:
      attestations: write
      contents: read
      id-token: write
      packages: write
    steps:

      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Install uv
        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
        with:
          version-file: "pyproject.toml"
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Install dev tools
        shell: bash
        run: .github/workflows/_install_dev_tools.bash

      - name: Install Python, venv and dependencies
        shell: bash
        run: uv sync --all-extras --frozen --link-mode=copy

      # Need xdisplay for testing QuPath app integration
      - name: Setup display
        uses: pyvista/setup-headless-display-action@5bc8de3bc71fcda7a96439571287a554901541a0 # v4.3

      - name: Print development version info
        if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
        shell: bash
        run: |
          TOML_VERSION=$(uv run python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
          echo "Development build - Current version in pyproject.toml: $TOML_VERSION"

      - name: Create .env file
        uses: SpicyPizza/create-envfile@ace6d4f5d7802b600276c23ca417e669f1a06f6f # v2.0.3
        with:
          # The following 3 lines are correct even if vscode complains
          envkey_AIGNOSTICS_API_ROOT: ${{ inputs.platform_environment == 'staging' && 'https://platform-staging.aignostics.com' || 'https://platform.aignostics.com' }}
          envkey_AIGNOSTICS_CLIENT_ID_DEVICE: ${{ inputs.platform_environment == 'staging' && secrets.AIGNOSTICS_CLIENT_ID_DEVICE_STAGING || secrets.AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION }}
          envkey_AIGNOSTICS_REFRESH_TOKEN: ${{ inputs.platform_environment == 'staging' && secrets.AIGNOSTICS_REFRESH_TOKEN_STAGING || secrets.AIGNOSTICS_REFRESH_TOKEN_PRODUCTION }}
          envkey_AIGNOSTICS_SENTRY_ENABLED: true
          envkey_AIGNOSTICS_SENTRY_DSN: ${{ secrets.SENTRY_DSN}}
          envkey_AIGNOSTICS_SENTRY_DEBUG: false
          envkey_AIGNOSTICS_SENTRY_SEND_DEFAULT_PII: false
          envkey_AIGNOSTICS_SENTRY_SAMPLE_RATE: 0
          envkey_AIGNOSTICS_SENTRY_TRACES_SAMPLE_RATE: 0
          envkey_AIGNOSTICS_SENTRY_PROFILES_SAMPLE_RATE: 0
          fail_on_empty: false

      - name: Set up GCP credentials for bucket access
        shell: bash
        env:
          GCP_CREDENTIALS: ${{ inputs.platform_environment == 'staging' && secrets.GCP_CREDENTIALS_STAGING || secrets.GCP_CREDENTIALS_PRODUCTION }}
        run: |
          echo "$GCP_CREDENTIALS" | base64 -d > credentials.json
          echo "GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/credentials.json" >> $GITHUB_ENV

      - name: Validate installation (single Python version)
        id: validate
        continue-on-error: true
        shell: bash
        run: |
            OUTPUT=$(uv run --no-dev aignostics --help)
            if [[ "$OUTPUT" != *"built with love in Berlin"* ]]; then
            echo "Output does not contain 'built with love in Berlin'"
            exit 1
            fi

      - name: Test / Smoke (single Python version)
        id: smoke
        continue-on-error: true
        shell: bash
        run: |
          uv run --no-dev aignostics --help
          uv run --all-extras aignostics system info
          uv run --all-extras aignostics system health
          uv run --all-extras aignostics user whoami --mask-secrets
          uv run --all-extras aignostics application list
          uv run --all-extras aignostics application run list --verbose --limit 1

      - name: Test / Unit (multiple Python versions)
        id: unit
        continue-on-error: true
        uses: ./.github/actions/run-tests
        with:
          test-type: unit
          make-target: test_unit_matrix
          skip-marker: skip:test:unit
          summary-title: All unit tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / Integration (multiple Python versions)
        id: integration
        continue-on-error: true
        uses: ./.github/actions/run-tests
        with:
          test-type: integration
          make-target: test_integration_matrix
          skip-marker: skip:test:integration
          summary-title: All integration tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / E2E / regular (multiple Python versions)
        id: e2e-regular
        continue-on-error: true
        uses: ./.github/actions/run-tests
        with:
          test-type: e2e
          make-target: test_e2e_matrix
          skip-marker: skip:test:e2e
          summary-title: All regular e2e tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / E2E / long running (single Python version)
        id: e2e-long-running
        continue-on-error: true
        if: |
          !github.event.pull_request.draft
        uses: ./.github/actions/run-tests
        with:
          test-type: long-running
          make-target: test_long_running
          skip-marker: skip:test:long_running
          summary-title: All long running e2e tests passed
          commit-message: ${{ inputs.commit_message }}

      - name: Test / E2E / very long running (single Python version)
        id: e2e-very-long-running
        continue-on-error: true
        uses: ./.github/actions/run-tests
        with:
          test-type: very-long-running
          make-target: test_very_long_running
          summary-title: All very long running e2e tests passed
 
      - name: Upload test artifacts for inspection
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: ${{ always() && (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') }}
        with:
          name: test-results-ubuntu-latest
          path: |
            reports/mypy_junit.xml
            reports/junit_*.xml
            reports/coverage.xml
            reports/coverage.md
            reports/coverage_html
          retention-days: 7
      - name: Determine overall test status
        id: test-status
        if: always()
        shell: bash
        run: |
          # Track which test types failed
          FAILED_TESTS=()
          EXIT_CODE=0
          
          if [ "${{ steps.validate.outcome }}" == "failure" ]; then
            FAILED_TESTS+=("validation")
            EXIT_CODE=1
          fi
          
          if [ "${{ steps.smoke.outcome }}" == "failure" ]; then
            FAILED_TESTS+=("smoke")
            EXIT_CODE=1
          fi
          
          if [ "${{ steps.unit.outcome }}" == "failure" ]; then
            FAILED_TESTS+=("unit")
            EXIT_CODE=1
          fi
          
          if [ "${{ steps.integration.outcome }}" == "failure" ]; then
            FAILED_TESTS+=("integration")
            EXIT_CODE=1
          fi
          
          if [ "${{ steps.e2e-regular.outcome }}" == "failure" ]; then
            FAILED_TESTS+=("e2e-regular")
            EXIT_CODE=1
          fi
          
          if [ "${{ steps.e2e-long-running.outcome }}" == "failure" ]; then
            FAILED_TESTS+=("e2e-long-running")
            EXIT_CODE=1
          fi
          
          if [ "${{ steps.e2e-very-long-running.outcome }}" == "failure" ]; then
            FAILED_TESTS+=("e2e-very-long-running")
            EXIT_CODE=1
          fi
          
          # Export for next steps
          echo "exit_code=${EXIT_CODE}" >> $GITHUB_OUTPUT
          echo "failed_tests=${FAILED_LIST}" >> $GITHUB_OUTPUT
          if [ ${#FAILED_TESTS[@]} -eq 0 ]; then
            echo "failed_tests=none" >> $GITHUB_OUTPUT
            echo "✅ All test types passed" >> $GITHUB_STEP_SUMMARY
          else
            FAILED_LIST=$(IFS=,; echo "${FAILED_TESTS[*]}")
            echo "failed_tests=${FAILED_LIST}" >> $GITHUB_OUTPUT
            echo "❌ Failed test types: ${FAILED_LIST}" >> $GITHUB_STEP_SUMMARY
          fi

      - name: Heartbeat
        if: always()
        env:
          BETTERSTACK_HEARTBEAT_URL: "${{ inputs.platform_environment == 'staging' && secrets.BETTERSTACK_HEARTBEAT_URL_FLOWS_STAGING || secrets.BETTERSTACK_HEARTBEAT_URL_FLOWS_PRODUCTION }}"
          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
        shell: bash
        run: |
          EXIT_CODE=${{ steps.test-status.outputs.exit_code }}
          FAILED_TESTS="${{ steps.test-status.outputs.failed_tests }}"
          
          # Send heartbeat to Sentry, defining the schedule on the fly
          SENTRY_EXIT_CODE=$(sentry-cli monitors run -e CI --schedule "0 12 * * *" --check-in-margin 30 --max-runtime 1 scheduled-testing-${{ inputs.platform_environment }}-hourly --timezone "Europe/Berlin" -- sh -c "exit $EXIT_CODE")

          # Provide heartbeat to BetterStack for monitoring/alerting if heartbeat url is configured as secret
          if [ -n "$BETTERSTACK_HEARTBEAT_URL" ]; then
            BETTERSTACK_METADATA_PAYLOAD=$(jq -n \
              --arg github_workflow "${{ github.workflow }}" \
              --arg github_run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
              --arg github_run_id "${{ github.run_id }}" \
              --arg github_job "${{ github.job }}" \
              --arg github_sha "${{ github.sha }}" \
              --arg github_actor "${{ github.actor }}" \
              --arg github_repository "${{ github.repository }}" \
              --arg github_ref "${{ github.ref }}" \
              --arg job_status "${{ job.status }}" \
              --arg github_event_name "${{ github.event_name }}" \
              --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
              --arg failed_tests "${FAILED_TESTS}" \
              --arg validate_status "${{ steps.validate.outcome }}" \
              --arg smoke_status "${{ steps.smoke.outcome }}" \
              --arg unit_status "${{ steps.unit.outcome }}" \
              --arg integration_status "${{ steps.integration.outcome }}" \
              --arg e2e_regular_status "${{ steps.e2e-regular.outcome }}" \
              --arg e2e_long_running_status "${{ steps.e2e-long-running.outcome }}" \
              --arg e2e_very_long_running_status "${{ steps.e2e-very-long-running.outcome }}" \
              '{
                github: {
                  workflow: $github_workflow,
                  run_url: $github_run_url,
                  run_id: $github_run_id,
                  job: $github_job,
                  sha: $github_sha,
                  actor: $github_actor,
                  repository: $github_repository,
                  ref: $github_ref,
                  event_name: $github_event_name
                },
                job: {
                  status: $job_status,
                },
                tests: {
                  failed: $failed_tests,
                  validation: $validate_status,
                  smoke: $smoke_status,
                  unit: $unit_status,
                  integration: $integration_status,
                  e2e_regular: $e2e_regular_status,
                  e2e_long_runnin: $e2e_long_running_status,
                  e2e_very_long_running: $e2e_very_long_running_status
                },
                timestamp: $timestamp,
              }'
            )
            curl \
              --fail-with-body \
              --silent \
              --request POST \
              --header "Content-Type: application/json" \
              --data-binary "${BETTERSTACK_METADATA_PAYLOAD}" \
              "${BETTERSTACK_HEARTBEAT_URL}/${EXIT_CODE}"
            echo "INFO: Sent heartbeat to betterstack with exit code '${EXIT_CODE}' and failed tests: '${FAILED_TESTS}'"
          else
            echo "WARNING: No BetterStack heartbeat URL configured, skipped heartbeat notification."
          fi

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
        if: ${{ !cancelled() && (env.GITHUB_WORKFLOW_RUNTIME != 'ACT')}}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: aignostics/python-sdk

      - name: Upload test results to Codecov
        if: ${{ !cancelled() && (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') }}
        uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1.2.1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: SonarQube Scan
        if: ${{ !cancelled() && (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') }}
        uses: SonarSource/sonarqube-scan-action@299e4b793aaa83bf2aba7c9c14bedbb485688ec4 # v7.1.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

      - name: Fail workflow if any tests failed
        if: always()
        shell: bash
        run: |
          EXIT_CODE=${{ steps.test-status.outputs.exit_code }}
          if [ "$EXIT_CODE" != "0" ]; then
            echo "❌ Workflow failed due to test failures: ${{ steps.test-status.outputs.failed_tests }}"
            exit $EXIT_CODE
          fi