E2E AutoTest #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E AutoTest | |
| on: | |
| schedule: | |
| # Every weekday (Mon–Fri) at 13:00 Shanghai time (05:00 UTC) | |
| - cron: '0 5 * * 1-5' | |
| workflow_dispatch: | |
| inputs: | |
| test_plan: | |
| description: "Test plan to run (leave empty for all)" | |
| required: false | |
| default: "" | |
| type: string | |
| vsix_urls: | |
| description: "VSIX URLs or GitHub release tag URLs, comma-separated (e.g. https://github.com/redhat-developer/vscode-java/releases/tag/v1.54.0,https://host/vscode-java-debug-0.58.0.vsix). Release tag URLs auto-resolve to the platform-specific VSIX for the runner OS." | |
| required: false | |
| default: "" | |
| type: string | |
| pre_release: | |
| description: "Install pre-release versions of marketplace extensions" | |
| required: false | |
| default: true | |
| type: boolean | |
| jobs: | |
| # ── Job 1: Discover test plans ────────────────────────── | |
| discover: | |
| if: ${{ inputs.test_plan == '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.scan.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Scan test plans | |
| id: scan | |
| run: | | |
| plans=$(ls test-plans/*.yaml | xargs -I{} basename {} .yaml | grep -v java-fresh-import | jq -R . | jq -sc .) | |
| echo "matrix=$plans" >> "$GITHUB_OUTPUT" | |
| echo "Found plans: $plans" | |
| # ── Job 2: Run each test plan in parallel ─────────────── | |
| e2e-test: | |
| if: ${{ inputs.test_plan == '' }} | |
| needs: discover | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| plan: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Clone vscode-java (test projects) | |
| run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java | |
| - name: Clone eclipse.jdt.ls (Gradle test projects) | |
| run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup Java 25 (for java25 test plans) | |
| if: contains(matrix.plan, 'java25') | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25-ea | |
| - name: Create JDK 25 path | |
| if: contains(matrix.plan, 'java25') | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME | |
| - name: Setup Java 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Download VSIX files | |
| if: ${{ inputs.vsix_urls != '' }} | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Directory -Path vsix -Force | Out-Null | |
| $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" } | |
| # Map runner OS/arch to vscode-java platform identifiers | |
| $platformMap = @{ "Windows" = "win32"; "Linux" = "linux"; "macOS" = "darwin" } | |
| $archMap = @{ "X64" = "x64"; "ARM64" = "arm64" } | |
| $platform = $platformMap["${{ runner.os }}"] | |
| $arch = $archMap["${{ runner.arch }}"] | |
| $platformId = "$platform-$arch" | |
| Write-Host "Runner platform: $platformId (${{ runner.os }}/${{ runner.arch }})" | |
| $resolvedUrls = @() | |
| foreach ($url in $urls) { | |
| if ($url -match '^https://github\.com/([^/]+)/([^/]+)/releases/tag/(.+)$') { | |
| $owner = $Matches[1]; $repo = $Matches[2]; $tag = $Matches[3] | |
| Write-Host "Resolving GitHub release: $owner/$repo@$tag for platform $platformId" | |
| $apiUrl = "https://api.github.com/repos/$owner/$repo/releases/tags/$tag" | |
| $release = Invoke-RestMethod -Uri $apiUrl -Headers @{ Accept = "application/vnd.github.v3+json" } -UseBasicParsing | |
| $platformAsset = $release.assets | Where-Object { $_.name -like "*-$platformId-*" -and $_.name -like "*.vsix" } | Select-Object -First 1 | |
| if ($platformAsset) { | |
| Write-Host " Found platform-specific VSIX: $($platformAsset.name)" | |
| $resolvedUrls += $platformAsset.browser_download_url | |
| } else { | |
| $universalAsset = $release.assets | Where-Object { $_.name -notmatch '-(darwin|linux|win32)-' -and $_.name -like "*.vsix" } | Select-Object -First 1 | |
| if ($universalAsset) { | |
| Write-Host " No platform-specific VSIX found, using universal: $($universalAsset.name)" | |
| $resolvedUrls += $universalAsset.browser_download_url | |
| } else { | |
| Write-Host "::warning::No matching VSIX found in release $owner/$repo@$tag for platform $platformId" | |
| } | |
| } | |
| } else { | |
| $resolvedUrls += $url | |
| } | |
| } | |
| foreach ($url in $resolvedUrls) { | |
| $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0]) | |
| Write-Host "Downloading: $url → vsix/$fileName" | |
| Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing | |
| } | |
| Write-Host "Downloaded VSIX files:" | |
| Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { Write-Host " $($_.Name) ($([math]::Round($_.Length/1MB, 1)) MB)" } | |
| - name: Run ${{ matrix.plan }} | |
| shell: pwsh | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: | | |
| $autotestArgs = @("run", "test-plans/${{ matrix.plan }}.yaml") | |
| if (Test-Path vsix) { | |
| $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join "," | |
| if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) } | |
| } | |
| if ("${{ inputs.pre_release }}" -ne "false") { $autotestArgs += "--pre-release" } | |
| Write-Host "Running: autotest $($autotestArgs -join ' ')" | |
| & autotest @autotestArgs | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.plan }} | |
| path: test-results/ | |
| retention-days: 30 | |
| # ── Job 2b: Run a single test plan (when specified) ───── | |
| e2e-single: | |
| if: ${{ inputs.test_plan != '' }} | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Clone vscode-java | |
| run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java | |
| - name: Clone eclipse.jdt.ls | |
| run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup Java 25 | |
| if: contains(inputs.test_plan, 'java25') | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25-ea | |
| - name: Create JDK 25 path | |
| if: contains(inputs.test_plan, 'java25') | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME | |
| - name: Setup Java 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Download VSIX files | |
| if: ${{ inputs.vsix_urls != '' }} | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Directory -Path vsix -Force | Out-Null | |
| $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" } | |
| # Map runner OS/arch to vscode-java platform identifiers | |
| $platformMap = @{ "Windows" = "win32"; "Linux" = "linux"; "macOS" = "darwin" } | |
| $archMap = @{ "X64" = "x64"; "ARM64" = "arm64" } | |
| $platform = $platformMap["${{ runner.os }}"] | |
| $arch = $archMap["${{ runner.arch }}"] | |
| $platformId = "$platform-$arch" | |
| Write-Host "Runner platform: $platformId (${{ runner.os }}/${{ runner.arch }})" | |
| $resolvedUrls = @() | |
| foreach ($url in $urls) { | |
| if ($url -match '^https://github\.com/([^/]+)/([^/]+)/releases/tag/(.+)$') { | |
| $owner = $Matches[1]; $repo = $Matches[2]; $tag = $Matches[3] | |
| Write-Host "Resolving GitHub release: $owner/$repo@$tag for platform $platformId" | |
| $apiUrl = "https://api.github.com/repos/$owner/$repo/releases/tags/$tag" | |
| $release = Invoke-RestMethod -Uri $apiUrl -Headers @{ Accept = "application/vnd.github.v3+json" } -UseBasicParsing | |
| $platformAsset = $release.assets | Where-Object { $_.name -like "*-$platformId-*" -and $_.name -like "*.vsix" } | Select-Object -First 1 | |
| if ($platformAsset) { | |
| Write-Host " Found platform-specific VSIX: $($platformAsset.name)" | |
| $resolvedUrls += $platformAsset.browser_download_url | |
| } else { | |
| $universalAsset = $release.assets | Where-Object { $_.name -notmatch '-(darwin|linux|win32)-' -and $_.name -like "*.vsix" } | Select-Object -First 1 | |
| if ($universalAsset) { | |
| Write-Host " No platform-specific VSIX found, using universal: $($universalAsset.name)" | |
| $resolvedUrls += $universalAsset.browser_download_url | |
| } else { | |
| Write-Host "::warning::No matching VSIX found in release $owner/$repo@$tag for platform $platformId" | |
| } | |
| } | |
| } else { | |
| $resolvedUrls += $url | |
| } | |
| } | |
| foreach ($url in $resolvedUrls) { | |
| $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0]) | |
| Write-Host "Downloading: $url → vsix/$fileName" | |
| Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing | |
| } | |
| Write-Host "Downloaded VSIX files:" | |
| Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { Write-Host " $($_.Name) ($([math]::Round($_.Length/1MB, 1)) MB)" } | |
| - name: Run ${{ inputs.test_plan }} | |
| shell: pwsh | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: | | |
| $autotestArgs = @("run", "test-plans/${{ inputs.test_plan }}") | |
| if (Test-Path vsix) { | |
| $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join "," | |
| if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) } | |
| } | |
| if ("${{ inputs.pre_release }}" -ne "false") { $autotestArgs += "--pre-release" } | |
| Write-Host "Running: autotest $($autotestArgs -join ' ')" | |
| & autotest @autotestArgs | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-test-results | |
| path: test-results/ | |
| retention-days: 30 | |
| # ── Job 3: Aggregate analysis ─────────────────────────── | |
| analyze: | |
| if: ${{ always() && inputs.test_plan == '' }} | |
| needs: e2e-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Download all results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-* | |
| path: all-results | |
| merge-multiple: false | |
| - name: Organize results | |
| run: | | |
| mkdir -p test-results | |
| for dir in all-results/results-*/; do | |
| find "$dir" -name "results.json" -exec dirname {} \; | while read d; do | |
| name=$(basename "$d") | |
| cp -r "$d" "test-results/$name" | |
| done | |
| done | |
| echo "Organized results:" | |
| ls test-results/ | |
| - name: Analyze results | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: autotest analyze test-results --output test-results | |
| - name: Write Job Summary | |
| if: always() | |
| run: | | |
| if [ -f test-results/summary.md ]; then | |
| cat test-results/summary.md >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Upload aggregate results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-aggregate-summary | |
| path: test-results/summary.md | |
| retention-days: 30 |