Repository navigation
607 lines (583 loc) · 29.2 KB
/
Copy pathrelease.yml
File metadata and controls
607 lines (583 loc) · 29.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Release workflow for pyspark-client-rust (the Rust crates on crates.io + the
# Python wheel on PyPI). Modeled on Apache Spark's release workflow: it is
# DISPATCHED FROM A COMMITTER'S FORK (which holds the publish tokens), takes the
# same inputs, and follows the same release-candidate-then-finalize flow.
#
# Inputs (identical to apache/spark's release workflow):
# branch apache/spark-connect-rust branch the release source is fetched
# from (always from apache, never the fork's own copy). Default
# master. Unused by finalize, which uses the RC tag as its source.
# release-version Final version, e.g. 4.2.0. Leave empty (with rc-count) for a dry
# run. Must equal the Cargo.toml/pyproject.toml version on `branch`
# (asserted) -- the RC number never appears in the artifact version.
# rc-count RC number, e.g. 1. Leave empty (with release-version) for a dry run.
# finalize true => convert the named RC into the OFFICIAL release
# (IRREVERSIBLE). Default false.
#
# The three modes are derived from the inputs (see the `resolve` job):
#
# * Dry run (release-version AND rc-count empty): build every wheel + sdist and
# `cargo publish --dry-run`. Creates no tag, publishes nothing. This is the
# testable path and also runs on a schedule to keep the release path healthy.
#
# * Cut an RC (release-version + rc-count, finalize=false): build the wheels +
# sdist -- versioned as the FINAL release-version, since the RC number lives
# only in the tag/release name, never in the artifact version -- then
# auto-create and push the tag `v<version>-rc<rc>` to the canonical apache repo
# and attach the artifacts to a GitHub *pre-release* of that tag. crates.io/PyPI
# are NOT touched, so an RC stays fully deletable.
#
# * Finalize (finalize=true, naming an existing RC): check out the RC tag,
# `twine upload` the EXACT wheels+sdist from that RC's GitHub pre-release to
# PyPI, `cargo publish` the identical source to crates.io, push a permanent
# `v<version>` git tag marking the released commit, then delete every
# `v<version>-rc*` GitHub pre-release (and its tag) from the apache repo. GitHub
# Releases are only RC staging; the official release lives on PyPI + crates.io,
# marked in git by the immutable `v<version>` tag on the apache repo.
#
# The workflow RUNS in a committer's fork, but all tags and GitHub Releases land on
# the canonical apache repo (never the fork). Required fork secrets for a real run:
# - CARGO_REGISTRY_TOKEN crates.io publish
# - PYPI_API_TOKEN PyPI publish
# - RELEASE_GITHUB_TOKEN a Personal Access Token with write (contents) access to
# apache/spark-connect-rust, used to push the tags and
# create/delete the GitHub Releases there. The built-in
# GITHUB_TOKEN can only write to the fork, so it is NOT used
# for these; it is read-only here.
# Without these secrets, only dry runs work.
#
# To drop an RC without finalizing: delete its GitHub pre-release and the
# `v<version>-rc<n>` tag from the apache repo (nothing was published to PyPI/crates.io).
name: Release
on:
schedule:
# Periodic dry run (runs only in the repo hosting the workflow; GitHub does not
# run scheduled workflows in forks), to catch release-path breakage early.
- cron: '0 7 */2 * *'
workflow_dispatch:
inputs:
branch:
description: 'Branch to release. Leave release-version/rc-count empty to launch a dryrun. Dispatch this workflow only in the forked repository.'
required: true
default: master
release-version:
description: 'Release version (e.g. 4.2.0). Leave it empty to launch a dryrun.'
required: false
rc-count:
description: 'RC number (e.g. 1). Leave it empty to launch a dryrun.'
required: false
finalize:
description: 'Whether to convert RC to the official release (IRREVERSIBLE)'
required: true
type: boolean
default: false
permissions:
contents: read
# Only one release may run at a time, and a publish in flight is never cancelled
# (cancelling mid-publish is what leaves crates.io/PyPI half-updated).
concurrency:
group: release
cancel-in-progress: false
env:
# The release source is always fetched from this canonical apache repository.
UPSTREAM_REPO: apache/spark-connect-rust
jobs:
# A real (RC/finalize) run must come from a committer's fork, which holds the publish
# tokens; the canonical apache repo only does dry runs. `resolve`'s `if:` already skips
# real runs there, but a bare skip is a green, all-skipped no-op that explains nothing.
# This job runs ONLY in that mistaken case and fails loudly with instructions.
reject-real-release-from-apache:
name: Reject real releases dispatched from the apache repo
runs-on: ubuntu-latest
if: >-
github.repository == 'apache/spark-connect-rust' &&
(inputs.release-version != '' || inputs.rc-count != '')
steps:
- run: |
echo "::error::Real releases (RC or finalize) must be dispatched from a committer's FORK of spark-connect-rust (which holds CARGO_REGISTRY_TOKEN / PYPI_API_TOKEN) — not from apache/spark-connect-rust. This canonical repository only runs dry runs: leave release-version and rc-count empty. See the workflow header for the fork-based release process."
exit 1
resolve:
name: Resolve release mode
runs-on: ubuntu-latest
# In the canonical apache repo, only dry runs are allowed (release-version and
# rc-count empty). A fork may perform real runs. This mirrors apache/spark.
# NOTE: the `env` context is not available in a job-level `if:`, so the canonical
# repo is spelled out literally here; it must match `env.UPSTREAM_REPO` below.
if: >-
github.repository != 'apache/spark-connect-rust' ||
(inputs.release-version == '' && inputs.rc-count == '')
outputs:
repo: ${{ env.UPSTREAM_REPO }}
mode: ${{ steps.r.outputs.mode }}
version: ${{ steps.r.outputs.version }}
rc_tag: ${{ steps.r.outputs.rc_tag }}
branch: ${{ steps.r.outputs.branch }}
# The branch resolved to a single immutable commit SHA, checked out everywhere
# downstream so the wheels, sdist, and RC tag can never come from different commits.
sha: ${{ steps.r.outputs.sha }}
steps:
- name: Determine mode and validate inputs
id: r
env:
IN_BRANCH: ${{ inputs.branch }}
IN_VERSION: ${{ inputs.release-version }}
IN_RC: ${{ inputs.rc-count }}
IN_FINALIZE: ${{ inputs.finalize }}
run: |
set -euo pipefail
BRANCH="${IN_BRANCH:-master}"
RV="${IN_VERSION:-}"
RC="${IN_RC:-}"
FIN="${IN_FINALIZE:-false}"
# release-version and rc-count must be provided together (or neither).
if { [ -n "$RV" ] && [ -z "$RC" ]; } || { [ -z "$RV" ] && [ -n "$RC" ]; }; then
echo "::error::Provide BOTH 'release-version' and 'rc-count', or leave BOTH empty for a dry run."
exit 1
fi
if [ "$FIN" = "true" ]; then
if [ -z "$RV" ] || [ -z "$RC" ]; then
echo "::error::finalize=true requires 'release-version' and 'rc-count' naming the RC to promote."
exit 1
fi
MODE=finalize
elif [ -n "$RV" ]; then
MODE=rc
else
MODE=dryrun
fi
RC_TAG=""
[ -n "$RV" ] && RC_TAG="v${RV}-rc${RC}"
# Pin the branch to one immutable commit SHA so every downstream checkout
# (wheels, sdist, RC tag) uses the SAME source even if the branch moves
# mid-run. finalize does not need it (it checks out the RC tag instead).
SHA=""
if [ "$MODE" = "rc" ] || [ "$MODE" = "dryrun" ]; then
SHA=$(git ls-remote "https://github.com/${UPSTREAM_REPO}.git" "refs/heads/${BRANCH}" | awk 'NR==1{print $1}')
if [ -z "$SHA" ]; then
echo "::error::Branch '${BRANCH}' not found on ${UPSTREAM_REPO}."
exit 1
fi
fi
{
echo "mode=$MODE"
echo "version=$RV"
echo "rc_tag=$RC_TAG"
echo "branch=$BRANCH"
echo "sha=$SHA"
} >> "$GITHUB_OUTPUT"
echo "Resolved: mode=$MODE version='${RV}' rc='${RC}' tag='${RC_TAG}' branch='${BRANCH}' sha='${SHA}'"
- name: Check out release source (for version validation)
if: steps.r.outputs.mode == 'dryrun' || steps.r.outputs.mode == 'rc'
uses: actions/checkout@v4
with:
repository: ${{ env.UPSTREAM_REPO }}
ref: ${{ steps.r.outputs.sha }}
- name: Validate versions
# Runs for dry run AND rc. The Cargo<->pyproject agreement and the path-dep pin
# checks always run (so the scheduled dry run actually exercises them); the
# release-version equality check is additional and rc-only.
if: steps.r.outputs.mode == 'dryrun' || steps.r.outputs.mode == 'rc'
env:
MODE: ${{ steps.r.outputs.mode }}
V: ${{ steps.r.outputs.version }}
run: |
set -euo pipefail
CARGO_VER=$(grep -m1 '^version' Cargo.toml | awk -F'"' '{print $2}')
PYPROJECT_VER=$(grep -m1 '^version' pyproject.toml | awk -F'"' '{print $2}')
echo "mode=${MODE} Cargo=${CARGO_VER} pyproject=${PYPROJECT_VER} release-version=${V:-<none>}"
# 1. Cargo.toml and pyproject.toml must declare the same version.
if [ "$CARGO_VER" != "$PYPROJECT_VER" ]; then
echo "::error::Cargo.toml (${CARGO_VER}) and pyproject.toml (${PYPROJECT_VER}) versions disagree."
exit 1
fi
# 2. Internal path-dep pins (e.g. spark-connect-core -> proto) must track the
# workspace version, or `cargo publish` embeds a stale hard dep into the
# published crate and breaks downstream consumers.
BAD=$(grep -rEn 'path = "\.\./spark-connect[^"]*"' crates/*/Cargo.toml \
| grep -E 'version = "' \
| grep -v "version = \"${CARGO_VER}\"" || true)
if [ -n "$BAD" ]; then
echo "::error::internal path-dep version pin(s) do not match the workspace version ${CARGO_VER}:"
echo "$BAD"
exit 1
fi
# 3. For an RC, the declared version must equal the requested release-version.
if [ "$MODE" = "rc" ] && [ "$CARGO_VER" != "$V" ]; then
echo "::error::Branch '${{ steps.r.outputs.branch }}' declares ${CARGO_VER}, but release-version=${V}. Bump the branch to ${V} (Cargo.toml, pyproject.toml, and the internal path-dep pins) before cutting the RC."
exit 1
fi
# ---- Build (dry run + RC only; finalize reuses the RC's already-built artifacts) ----
build-wheels:
name: Build wheel (${{ matrix.target }})
needs: resolve
if: needs.resolve.outputs.mode == 'dryrun' || needs.resolve.outputs.mode == 'rc'
strategy:
fail-fast: false
matrix:
include:
- { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, zig: true }
- { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, zig: true }
- { os: macos-latest, target: universal2-apple-darwin }
- { os: windows-latest, target: x86_64-pc-windows-msvc }
uses: ./.github/workflows/build-wheel.yml
with:
runner: ${{ matrix.os }}
target: ${{ matrix.target }}
zig: ${{ matrix.zig || false }}
# Preserve the previous release job's six-hour limit.
timeout_minutes: 360
# Keep the release source canonical and pinned by resolve, including when
# the workflow is dispatched from a committer's fork.
source_repository: ${{ needs.resolve.outputs.repo }}
source_ref: ${{ needs.resolve.outputs.sha }}
build-sdist:
name: Build source distribution
needs: resolve
if: needs.resolve.outputs.mode == 'dryrun' || needs.resolve.outputs.mode == 'rc'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
repository: ${{ env.UPSTREAM_REPO }}
ref: ${{ needs.resolve.outputs.sha }}
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Build sdist with maturin
run: |
python -m pip install --upgrade "maturin>=1.0,<2.0"
maturin sdist --out dist
- name: Verify declared License-File(s) exist in the sdist
# PyPI (PEP 639) rejects an sdist whose PKG-INFO declares a `License-File`
# that is not actually inside the tarball. maturin bundles the license into
# the wheel automatically but not the sdist, so this is easy to miss until
# the upload fails at finalize. Catch it here (dry run + RC), before publish.
run: |
python - <<'PY'
import glob, re, sys, tarfile
sdist = glob.glob('dist/*.tar.gz')[0]
t = tarfile.open(sdist)
names = set(t.getnames())
top = t.getnames()[0].split('/')[0]
pkg = t.extractfile(f'{top}/PKG-INFO').read().decode()
declared = re.findall(r'^License-File:\s*(.+)$', pkg, re.M)
missing = [lf for lf in declared if f'{top}/{lf}' not in names]
if missing:
print(f"::error::sdist {sdist} declares License-File {missing} but the "
f"file(s) are absent from the tarball; PyPI will reject it (PEP 639). "
f"Add them to [tool.maturin] include with format = \"sdist\".")
sys.exit(1)
print(f"sdist License-File check passed ({sdist}): {declared or 'none declared'}")
PY
- uses: actions/upload-artifact@v4
with:
name: wheels-sdist
path: dist/*.tar.gz
# ---- Dry run: rehearse the crates.io packaging without publishing ----
dryrun-crates:
name: Dry-run crates.io packaging
needs: resolve
if: needs.resolve.outputs.mode == 'dryrun'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
repository: ${{ env.UPSTREAM_REPO }}
ref: ${{ needs.resolve.outputs.sha }}
- name: Install Rust toolchain
run: |
rustup toolchain install stable --profile minimal
rustup default stable
- name: Install protoc
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Verify every published crate ships LICENSE and NOTICE
# ASF releases carry LICENSE and NOTICE in every artifact. cargo packages only the files
# under each crate's directory, so each crate links the root ones; `cargo package --list`
# shows whether the packaged crate has them.
run: |
set -euo pipefail
for c in apache-spark-connect-proto apache-spark-connect-core apache-spark-connect \
apache-spark-connect-macros apache-spark-connect-build; do
files=$(cargo package --list --allow-dirty -p "$c")
for f in LICENSE NOTICE; do
if ! grep -qx "$f" <<<"$files"; then
echo "::error::the crate $c does not package $f"
exit 1
fi
done
done
echo "Every crate packages LICENSE and NOTICE."
- name: cargo publish --dry-run (dependency-ordered)
run: |
set -euo pipefail
cargo publish --dry-run \
-p apache-spark-connect-proto \
-p apache-spark-connect-core \
-p apache-spark-connect \
-p apache-spark-connect-macros \
-p apache-spark-connect-build
# ---- Cut an RC: tag + GitHub pre-release with the artifacts (no crates.io/PyPI) ----
publish-rc:
name: Cut RC GitHub pre-release
needs: [resolve, build-wheels, build-sdist]
if: needs.resolve.outputs.mode == 'rc'
runs-on: ubuntu-latest
permissions:
# The tag push + GitHub Release go to the canonical apache repo via
# RELEASE_GITHUB_TOKEN; the built-in token only needs read (checkout/artifacts).
contents: read
steps:
- uses: actions/checkout@v4
with:
repository: ${{ env.UPSTREAM_REPO }}
ref: ${{ needs.resolve.outputs.sha }}
fetch-depth: 0 # full history so the tag push carries all reachable objects
# Do NOT persist the built-in GITHUB_TOKEN as an http.extraheader: it would
# override the RELEASE_GITHUB_TOKEN embedded in the push URL below, so the push
# to apache would go out as github-actions[bot] (which cannot write to apache).
persist-credentials: false
- uses: actions/download-artifact@v4
with:
pattern: wheels-*
path: dist
merge-multiple: true
- name: Create and push the RC tag to the canonical apache repo
env:
GH_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
TAG: ${{ needs.resolve.outputs.rc_tag }}
VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
git config user.name "${{ github.actor }}"
git config user.email "${{ github.actor }}@users.noreply.github.com"
if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then
echo "Local tag ${TAG} already exists; reusing."
else
git tag -a "${TAG}" -m "Release candidate ${TAG} of pyspark-client-rust ${VERSION}"
fi
# Push the RC tag (and the objects it reaches) to the canonical apache repo,
# so the GitHub pre-release below can hang off it. Force so re-cutting the same
# RC is idempotent.
git push --force "https://x-access-token:${GH_TOKEN}@github.com/${UPSTREAM_REPO}.git" "refs/tags/${TAG}"
- name: Create the GitHub pre-release with the artifacts
env:
GH_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
TAG: ${{ needs.resolve.outputs.rc_tag }}
VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
ls -l dist
NOTES="Release candidate ${TAG} of pyspark-client-rust ${VERSION}."$'\n\n'"> Not published to PyPI or crates.io. Install directly from a wheel/sdist asset below, e.g. \`pip install <asset-url>\`. This pre-release is deleted when the RC is finalized (or dropped)."
# Idempotent: a re-run updates the existing release and clobbers its assets.
if gh release view "${TAG}" --repo "${UPSTREAM_REPO}" >/dev/null 2>&1; then
gh release upload "${TAG}" dist/* --repo "${UPSTREAM_REPO}" --clobber
else
gh release create "${TAG}" dist/* --repo "${UPSTREAM_REPO}" \
--prerelease --title "${TAG}" --notes "${NOTES}"
fi
# ---- Finalize: promote the named RC into the official release (IRREVERSIBLE) ----
finalize:
name: Finalize release to PyPI + crates.io (IRREVERSIBLE)
needs: resolve
if: needs.resolve.outputs.mode == 'finalize'
runs-on: ubuntu-latest
permissions:
# All writes (permanent tag push, RC pre-release/tag deletion) go to the
# canonical apache repo via RELEASE_GITHUB_TOKEN; the built-in token only reads.
contents: read
steps:
- name: Abort window
run: |
echo "=============================================================================="
echo " CONVERTING RC ${{ needs.resolve.outputs.rc_tag }} INTO THE OFFICIAL RELEASE"
echo " ${{ needs.resolve.outputs.version }} -> PyPI + crates.io. THIS IS IRREVERSIBLE."
echo " Cancel this workflow now if you did not intend to finalize."
echo " Continuing in 60 seconds..."
echo "=============================================================================="
sleep 60
- name: Verify the RC pre-release exists
env:
GH_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
TAG: ${{ needs.resolve.outputs.rc_tag }}
run: |
set -euo pipefail
if ! gh release view "${TAG}" --repo "${UPSTREAM_REPO}" >/dev/null 2>&1; then
echo "::error::RC GitHub pre-release ${TAG} not found on ${UPSTREAM_REPO}. Cut the RC first."
exit 1
fi
- name: Check out the RC tag (the exact released source)
uses: actions/checkout@v4
with:
# The RC tag lives on the canonical apache repo, where the RC was cut.
repository: ${{ env.UPSTREAM_REPO }}
ref: ${{ needs.resolve.outputs.rc_tag }}
fetch-depth: 0
# Same reason as publish-rc: don't let the built-in token's persisted
# http.extraheader override RELEASE_GITHUB_TOKEN on the release-tag push below.
persist-credentials: false
- name: Validate the RC source versions
run: |
set -euo pipefail
V="${{ needs.resolve.outputs.version }}"
CARGO_VER=$(grep -m1 '^version' Cargo.toml | awk -F'"' '{print $2}')
PYPROJECT_VER=$(grep -m1 '^version' pyproject.toml | awk -F'"' '{print $2}')
echo "RC tag source: Cargo=${CARGO_VER} pyproject=${PYPROJECT_VER}; release-version=${V}"
# Cargo and pyproject must both equal the release-version (agreement + expected value).
if [ "$CARGO_VER" != "$V" ] || [ "$PYPROJECT_VER" != "$V" ]; then
echo "::error::RC tag ${{ needs.resolve.outputs.rc_tag }} declares Cargo=${CARGO_VER} / pyproject=${PYPROJECT_VER}, not ${V}."
exit 1
fi
# Path-dep pins must match, or `cargo publish` ships a stale hard dep (same check as resolve).
BAD=$(grep -rEn 'path = "\.\./spark-connect[^"]*"' crates/*/Cargo.toml \
| grep -E 'version = "' \
| grep -v "version = \"${CARGO_VER}\"" || true)
if [ -n "$BAD" ]; then
echo "::error::internal path-dep version pin(s) in the RC source do not match ${CARGO_VER}:"
echo "$BAD"
exit 1
fi
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Download the RC artifacts (the exact tested wheels + sdist)
env:
GH_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
TAG: ${{ needs.resolve.outputs.rc_tag }}
run: |
set -euo pipefail
mkdir -p dist
gh release download "${TAG}" --repo "${UPSTREAM_REPO}" --dir dist --pattern '*'
ls -l dist
- name: Install Rust toolchain
run: |
rustup toolchain install stable --profile minimal
rustup default stable
- name: Install protoc
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Verify every published crate ships LICENSE and NOTICE
# ASF releases carry LICENSE and NOTICE in every artifact. cargo packages only the files
# under each crate's directory, so each crate links the root ones; `cargo package --list`
# shows whether the packaged crate has them.
run: |
set -euo pipefail
for c in apache-spark-connect-proto apache-spark-connect-core apache-spark-connect \
apache-spark-connect-macros apache-spark-connect-build; do
files=$(cargo package --list --allow-dirty -p "$c")
for f in LICENSE NOTICE; do
if ! grep -qx "$f" <<<"$files"; then
echo "::error::the crate $c does not package $f"
exit 1
fi
done
done
echo "Every crate packages LICENSE and NOTICE."
- name: Publish wheels + sdist to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
run: |
set -euo pipefail
python -m pip install --upgrade twine
# --skip-existing makes a re-run after a partial upload safe.
twine upload --skip-existing dist/*
- name: Publish crates to crates.io (from the identical RC source)
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: |
set -euo pipefail
# The macros and build crates are what a Rust UDF depends on (docs/udfs.md).
CRATES=(apache-spark-connect-proto apache-spark-connect-core apache-spark-connect
apache-spark-connect-macros apache-spark-connect-build)
VER="${{ needs.resolve.outputs.version }}"
# Skip any crate already on crates.io at this version so a re-run is idempotent.
pkgs=()
for c in "${CRATES[@]}"; do
if curl -sf -H "User-Agent: spark-connect-rust-release (dev@spark.apache.org)" \
"https://crates.io/api/v1/crates/${c}/${VER}" >/dev/null; then
echo "${c} ${VER} is already on crates.io — skipping"
else
pkgs+=(-p "$c")
fi
done
if [ ${#pkgs[@]} -eq 0 ]; then
echo "All crates already published at ${VER}; nothing to do."
else
# cargo publishes in dependency order and waits for the index between crates.
cargo publish "${pkgs[@]}"
fi
- name: Tag the released commit with the permanent v<version> tag
env:
GH_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
TAG: v${{ needs.resolve.outputs.version }}
VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
git config user.name "${{ github.actor }}"
git config user.email "${{ github.actor }}@users.noreply.github.com"
URL="https://x-access-token:${GH_TOKEN}@github.com/${UPSTREAM_REPO}.git"
# HEAD is the RC tag's commit (the exact released source), checked out above.
LOCAL_SHA=$(git rev-parse HEAD)
if ! git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then
git tag -a "${TAG}" -m "pyspark-client-rust ${VERSION}"
fi
# A release tag is immutable. Look the remote tag up EXPLICITLY: a network/auth
# failure then aborts under set -e (instead of being mistaken for "already
# exists"). Only an exact same-commit tag is a no-op; a tag already pointing at
# a DIFFERENT commit is a hard error (we must never silently move a release tag,
# nor report success while it marks the wrong source).
REMOTE_LS=$(git ls-remote --tags "${URL}" "refs/tags/${TAG}")
if [ -z "${REMOTE_LS}" ]; then
git push "${URL}" "refs/tags/${TAG}" # a push failure is fatal under set -e
echo "Pushed release tag ${TAG} -> ${LOCAL_SHA}."
else
# Annotated tags list a peeled '^{}' line with the commit; prefer it, else the ref line.
REMOTE_SHA=$(printf '%s\n' "${REMOTE_LS}" | awk '/\^\{\}$/{c=$1} END{if(c!="")print c}')
[ -z "${REMOTE_SHA}" ] && REMOTE_SHA=$(printf '%s\n' "${REMOTE_LS}" | awk 'NR==1{print $1}')
if [ "${REMOTE_SHA}" = "${LOCAL_SHA}" ]; then
echo "Release tag ${TAG} already points at ${LOCAL_SHA}; nothing to do."
else
echo "::error::Release tag ${TAG} already exists on ${UPSTREAM_REPO} at ${REMOTE_SHA}, but the release commit is ${LOCAL_SHA}. Refusing to move an immutable release tag."
exit 1
fi
fi
- name: Remove all RC GitHub pre-releases + tags for this version
env:
GH_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }}
VERSION: ${{ needs.resolve.outputs.version }}
run: |
set -euo pipefail
# Every v<version>-rcN pre-release is RC staging; the official release now lives
# on PyPI + crates.io, so delete them all (and their tags) from the apache repo.
# Escape dots: grep -E treats '.' as any char, so an unescaped 4.2.0 would also
# match tags like v4x2x0-rcN.
ESC_V="${VERSION//./\\.}"
# Fetch the tag list in its own command so a `gh` failure (rate limit/network)
# aborts loudly under `set -e`, instead of being swallowed by the `|| true` that
# is only meant to absorb grep's "no match" exit code.
ALL_TAGS=$(gh release list --repo "${UPSTREAM_REPO}" --limit 200 --json tagName --jq '.[].tagName')
mapfile -t RCS < <(printf '%s\n' "${ALL_TAGS}" | grep -E "^v${ESC_V}-rc[0-9]+$" || true)
if [ "${#RCS[@]}" -eq 0 ]; then
echo "No v${VERSION}-rc* GitHub releases to remove."
fi
for t in "${RCS[@]}"; do
echo "Deleting RC release + tag ${t}"
gh release delete "${t}" --repo "${UPSTREAM_REPO}" --cleanup-tag --yes
done