Outerloop Tests #1823
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Executes outerloop tests and provides a temporary workflow_dispatch path for | |
| # branch testing of the transient CI rerun workflow helper. | |
| # | |
| # COPILOT INSTRUCTIONS: | |
| # - Keep the 'paths:' list in sync across tests-outerloop.yml and | |
| # tests-quarantine.yml | |
| # - Validate that each path exists in the repository before adding or | |
| # updating the list | |
| # - No external YAML file is used—only the workflow YAMLs themselves | |
| # hold the list | |
| # | |
| name: Outerloop Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: 'Workflow mode to run' | |
| required: true | |
| default: outerloop-tests | |
| type: choice | |
| options: | |
| - outerloop-tests | |
| - transient-ci-rerun | |
| run_id: | |
| description: 'CI workflow run ID to inspect when mode=transient-ci-rerun' | |
| required: false | |
| type: number | |
| schedule: | |
| - cron: '0 2 * * *' # Daily at 02:00 UTC | |
| # TEMPORARILY DISABLED pull_request trigger due to #12143 (disk space issues): https://github.com/dotnet/aspire/issues/12143 | |
| # pull_request: | |
| # paths: | |
| # - '.github/actions/**' | |
| # - '.github/workflows/**' | |
| # - 'eng/**' | |
| # - '!eng/pipelines/**' | |
| # - '!eng/scripts/**' | |
| # - '!eng/*pack/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| outerloop_tests: | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.mode != 'transient-ci-rerun' }} | |
| uses: ./.github/workflows/specialized-test-runner.yml | |
| with: | |
| testRunnerName: "OuterloopTestRunsheetBuilder" | |
| attributeName: "OuterloopTest" | |
| extraRunSheetBuilderArgs: "-p:RunOuterloopTests=true" | |
| extraTestArgs: "--filter-trait outerloop=true" | |
| enablePlaywrightInstall: true | |
| analyze-transient-failures: | |
| name: Analyze transient CI failures | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'transient-ci-rerun' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| checks: read | |
| outputs: | |
| source_run_id: ${{ steps.analyze.outputs.source_run_id }} | |
| source_run_url: ${{ steps.analyze.outputs.source_run_url }} | |
| retryable_jobs: ${{ steps.analyze.outputs.retryable_jobs }} | |
| pull_request_numbers: ${{ steps.analyze.outputs.pull_request_numbers }} | |
| retryable_count: ${{ steps.analyze.outputs.retryable_count }} | |
| skipped_count: ${{ steps.analyze.outputs.skipped_count }} | |
| rerun_eligible: ${{ steps.analyze.outputs.rerun_eligible }} | |
| max_retryable_jobs: ${{ steps.analyze.outputs.max_retryable_jobs }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Analyze failed jobs | |
| id: analyze | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| MANUAL_RUN_ID: ${{ inputs.run_id }} | |
| with: | |
| script: | | |
| const rerunWorkflow = require('./.github/workflows/auto-rerun-transient-ci-failures.js'); | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const maxRetryableJobs = rerunWorkflow.defaultMaxRetryableJobs; | |
| async function paginate(route, parameters, selectItems) { | |
| const items = []; | |
| for (let page = 1; ; page++) { | |
| const response = await github.request(route, { | |
| ...parameters, | |
| per_page: 100, | |
| page, | |
| }); | |
| items.push(...selectItems(response.data)); | |
| if (!response.headers.link || !response.headers.link.includes('rel="next"')) { | |
| return items; | |
| } | |
| } | |
| } | |
| const runId = Number(process.env.MANUAL_RUN_ID); | |
| if (!Number.isInteger(runId) || runId <= 0) { | |
| throw new Error('transient-ci-rerun mode requires a valid run_id input.'); | |
| } | |
| async function listJobsForAttempt(runId, attemptNumber) { | |
| return paginate( | |
| 'GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}/jobs', | |
| { | |
| owner, | |
| repo, | |
| run_id: runId, | |
| attempt_number: attemptNumber, | |
| }, | |
| data => data.jobs || []); | |
| } | |
| async function listAnnotations(jobId) { | |
| try { | |
| return await paginate( | |
| 'GET /repos/{owner}/{repo}/check-runs/{check_run_id}/annotations', | |
| { | |
| owner, | |
| repo, | |
| check_run_id: jobId, | |
| }, | |
| data => Array.isArray(data) ? data : []); | |
| } | |
| catch (error) { | |
| core.warning(`Failed to list annotations for job ${jobId}: ${error.message}`); | |
| return []; | |
| } | |
| } | |
| async function getJobLogText(jobId) { | |
| try { | |
| const response = await fetch(`https://api.github.com/repos/${owner}/${repo}/actions/jobs/${jobId}/logs`, { | |
| headers: { | |
| authorization: `Bearer ${process.env.GITHUB_TOKEN}`, | |
| accept: 'application/vnd.github+json', | |
| 'x-github-api-version': '2022-11-28', | |
| }, | |
| }); | |
| if (!response.ok) { | |
| throw new Error(`HTTP ${response.status}`); | |
| } | |
| return await response.text(); | |
| } | |
| catch (error) { | |
| core.warning(`Failed to fetch logs for job ${jobId}: ${error.message}`); | |
| return ''; | |
| } | |
| } | |
| const workflowRunResponse = await github.rest.actions.getWorkflowRun({ | |
| owner, | |
| repo, | |
| run_id: runId, | |
| }); | |
| const workflowRun = workflowRunResponse.data; | |
| const sourceRunUrl = workflowRun.html_url || `https://github.com/${owner}/${repo}/actions/runs/${workflowRun.id}`; | |
| core.setOutput('source_run_id', String(workflowRun.id)); | |
| core.setOutput('source_run_url', sourceRunUrl); | |
| core.setOutput('max_retryable_jobs', String(maxRetryableJobs)); | |
| core.setOutput('retryable_jobs', '[]'); | |
| core.setOutput('pull_request_numbers', '[]'); | |
| core.setOutput('retryable_count', '0'); | |
| core.setOutput('skipped_count', '0'); | |
| core.setOutput('rerun_eligible', 'false'); | |
| if (workflowRun.name && workflowRun.name !== 'CI') { | |
| console.log(`Workflow run ${workflowRun.id} is '${workflowRun.name}', not 'CI'. Skipping.`); | |
| return; | |
| } | |
| const pullRequestNumbers = [...new Set((workflowRun.pull_requests || []) | |
| .map(pullRequest => pullRequest.number) | |
| .filter(Number.isInteger))]; | |
| core.setOutput('pull_request_numbers', JSON.stringify(pullRequestNumbers)); | |
| if (pullRequestNumbers.length === 0) { | |
| console.log('No pull request is associated with this workflow run. Skipping.'); | |
| return; | |
| } | |
| const jobs = await listJobsForAttempt(workflowRun.id, workflowRun.run_attempt); | |
| const { failedJobs, retryableJobs, skippedJobs } = await rerunWorkflow.analyzeFailedJobs({ | |
| jobs, | |
| getAnnotationsForJob: async job => listAnnotations(job.id), | |
| getJobLogTextForJob: async job => getJobLogText(job.id), | |
| }); | |
| core.setOutput('retryable_jobs', JSON.stringify(retryableJobs.map(job => ({ | |
| id: job.id, | |
| name: job.name, | |
| htmlUrl: job.htmlUrl, | |
| reason: job.reason, | |
| })))); | |
| core.setOutput('retryable_count', String(retryableJobs.length)); | |
| core.setOutput('skipped_count', String(skippedJobs.length)); | |
| const rerunEligible = rerunWorkflow.computeRerunEligibility({ | |
| dryRun: false, | |
| retryableCount: retryableJobs.length, | |
| maxRetryableJobs, | |
| }); | |
| core.setOutput('rerun_eligible', String(rerunEligible)); | |
| await rerunWorkflow.writeAnalysisSummary({ | |
| summary: core.summary, | |
| failedJobs, | |
| retryableJobs, | |
| skippedJobs, | |
| maxRetryableJobs, | |
| dryRun: false, | |
| rerunEligible, | |
| sourceRunUrl, | |
| }); | |
| if (retryableJobs.length === 0) { | |
| console.log('No retryable failed jobs were detected.'); | |
| } | |
| rerun-transient-failures: | |
| name: Rerun transient CI failures | |
| needs: [analyze-transient-failures] | |
| if: ${{ needs.analyze-transient-failures.outputs.rerun_eligible == 'true' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: write | |
| issues: write | |
| pull-requests: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Rerun matched jobs | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| env: | |
| RETRYABLE_JOBS: ${{ needs.analyze-transient-failures.outputs.retryable_jobs }} | |
| PULL_REQUEST_NUMBERS: ${{ needs.analyze-transient-failures.outputs.pull_request_numbers }} | |
| SOURCE_RUN_URL: ${{ needs.analyze-transient-failures.outputs.source_run_url }} | |
| with: | |
| script: | | |
| const rerunWorkflow = require('./.github/workflows/auto-rerun-transient-ci-failures.js'); | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const retryableJobs = JSON.parse(process.env.RETRYABLE_JOBS || '[]'); | |
| const pullRequestNumbers = JSON.parse(process.env.PULL_REQUEST_NUMBERS || '[]'); | |
| const sourceRunUrl = process.env.SOURCE_RUN_URL; | |
| if (retryableJobs.length === 0) { | |
| console.log('No retryable jobs were provided to the rerun job.'); | |
| return; | |
| } | |
| await rerunWorkflow.rerunMatchedJobs({ | |
| github, | |
| owner, | |
| repo, | |
| retryableJobs, | |
| pullRequestNumbers, | |
| summary: core.summary, | |
| sourceRunUrl, | |
| }); |