Repository navigation
196 lines (188 loc) · 7.58 KB
/
Copy pathpython-leg.yml
File metadata and controls
196 lines (188 loc) · 7.58 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
name: python leg
# THE PYTHON LEG (docs/PORT-PYTHON.md): the complete `make test-python` on the pinned
# interpreters, on the fleet's Linux runners. The pins are CPython 3.11.18 (the floor),
# CPython 3.15.2 (the newest stable release) and mypy 1.18.2, built and
# installed into dist/ by `make dist-python`; .github/actions/python restores
# them from a cache keyed on every version, sha256 and the analyzer's lock,
# and checks the restored tree against the pins before anything runs on it.
# No step runs an interpreter from PATH.
#
# The macOS leg runs nightly and on dispatch (python-leg-nightly.yml): on a
# hosted macOS runner the leg does not fit a push or pull request's two
# minutes (internal/ci/nightly_test.go).
#
# The fleet has a two-minute per-commit cap, so its jobs split the same make
# prerequisites and enumerate every package test into its own row. Nightly
# still runs the unsplit make target on macOS.
on:
push:
branches: [ main, dev ]
paths:
- 'make/python*'
- 'tools/python-ci-*.sh'
- '.github/workflows/python-leg.yml'
- '.github/actions/python/**'
- 'third_party/serialize.py/**'
- 'compiler/*python*'
- 'internal/codegen/python/**'
- 'internal/codegen/pythontable/**'
- 'internal/tablenames/python.go'
- 'test/conformance/python/**'
- 'ir/**'
- 'internal/check/**'
pull_request:
paths:
- 'make/python*'
- 'tools/python-ci-*.sh'
- '.github/workflows/python-leg.yml'
- '.github/actions/python/**'
- 'third_party/serialize.py/**'
- 'compiler/*python*'
- 'internal/codegen/python/**'
- 'internal/codegen/pythontable/**'
- 'internal/tablenames/python.go'
- 'test/conformance/python/**'
- 'ir/**'
- 'internal/check/**'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || format('{0}-{1}', github.event_name, github.ref) }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
plan:
if: startsWith(github.head_ref, 'sprint/') == false && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
runs-on: [self-hosted, Linux, X64, fleet, schema]
timeout-minutes: 2
defaults:
run:
shell: go run ./tools/pgroup -- bash -e {0}
outputs:
rows0: ${{ steps.rows.outputs.rows0 }}
rows1: ${{ steps.rows.outputs.rows1 }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- id: rows
run: |
set -euo pipefail
rows=$(sh tools/python-ci-rows.sh)
rows0=$(printf '%s\n' "$rows" | sh tools/python-ci-partition.sh 0)
rows1=$(printf '%s\n' "$rows" | sh tools/python-ci-partition.sh 1)
echo "rows0=$rows0" >> "$GITHUB_OUTPUT"
echo "rows1=$rows1" >> "$GITHUB_OUTPUT"
fixed-corpus:
if: startsWith(github.head_ref, 'sprint/') == false && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
runs-on: [self-hosted, Linux, X64, fleet, schema]
timeout-minutes: 2
defaults:
run:
shell: go run ./tools/pgroup -- bash -e {0}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: build the reference corpus and generated Python units at this source
run: |
make serialize-siblings
sh tools/python-ci-corpus.sh build
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: python-fixed-corpus-${{ github.sha }}
path: build/python-ci-fixed-corpus.tar
if-no-files-found: error
overwrite: true
test:
name: python (${{ matrix.name }})
if: startsWith(github.head_ref, 'sprint/') == false && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
needs: [plan, fixed-corpus]
strategy:
fail-fast: false
max-parallel: 4
matrix:
include: ${{ fromJSON(needs.plan.outputs.rows0) }}
runs-on: [self-hosted, Linux, X64, fleet, schema]
timeout-minutes: 2
env:
GOMAXPROCS: '2'
defaults:
run:
shell: go run ./tools/pgroup -- bash -e {0}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: ./.github/actions/python
- name: stage the sibling runtimes
run: make serialize-siblings
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: matrix.corpus
with:
name: python-fixed-corpus-${{ github.sha }}
path: build
- name: verify the reference corpus source, runtime pin and bytes
if: matrix.corpus
run: sh tools/python-ci-corpus.sh restore
- name: run the selected complete-leg slice
env:
TARGET: ${{ matrix.target }}
PACKAGE: ${{ matrix.package }}
TEST: ${{ matrix.test }}
CORPUS: ${{ matrix.corpus }}
run: |
set -euo pipefail
if [ -n "$TARGET" ]; then
if [ "$CORPUS" = true ]; then
# The validated artifact supplies this prerequisite. Avoid
# rebuilding the C++ writer separately in every support row.
make -j"$(getconf _NPROCESSORS_ONLN)" --old-file=build/fixedform-corpus/.stamp --old-file=build/conformance-python "$TARGET"
else
make -j"$(getconf _NPROCESSORS_ONLN)" "$TARGET"
fi
else
SCHEMA_SLOW=1 go test -count=1 -tags schema_leg_python -run "^${TEST}$" "$PACKAGE"
fi
test-second:
name: python (${{ matrix.name }})
if: startsWith(github.head_ref, 'sprint/') == false && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
needs: [plan, fixed-corpus]
strategy:
fail-fast: false
max-parallel: 4
matrix:
include: ${{ fromJSON(needs.plan.outputs.rows1) }}
runs-on: [self-hosted, Linux, X64, fleet, schema]
timeout-minutes: 2
env:
GOMAXPROCS: '2'
defaults:
run:
shell: go run ./tools/pgroup -- bash -e {0}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: ./.github/actions/python
- name: stage the sibling runtimes
run: make serialize-siblings
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: matrix.corpus
with:
name: python-fixed-corpus-${{ github.sha }}
path: build
- name: verify the reference corpus source, runtime pin and bytes
if: matrix.corpus
run: sh tools/python-ci-corpus.sh restore
- name: run the selected complete-leg slice
env:
TARGET: ${{ matrix.target }}
PACKAGE: ${{ matrix.package }}
TEST: ${{ matrix.test }}
CORPUS: ${{ matrix.corpus }}
run: |
set -euo pipefail
if [ -n "$TARGET" ]; then
if [ "$CORPUS" = true ]; then
# The validated artifact supplies this prerequisite. Avoid
# rebuilding the C++ writer separately in every support row.
make -j"$(getconf _NPROCESSORS_ONLN)" --old-file=build/fixedform-corpus/.stamp --old-file=build/conformance-python "$TARGET"
else
make -j"$(getconf _NPROCESSORS_ONLN)" "$TARGET"
fi
else
SCHEMA_SLOW=1 go test -count=1 -tags schema_leg_python -run "^${TEST}$" "$PACKAGE"
fi