From a821a970638abe04991f95c210028b2296993b40 Mon Sep 17 00:00:00 2001 From: Jeremy White <44277022+jwhite242@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:31:46 -0700 Subject: [PATCH] Finalize the 1.2.1 version for release --- CHANGELOG.md | 23 ++ CONTRIBUTING.md | 4 +- RELEASE.md | 299 ++++++++++++++++++ ...20260408_182201_jwhite242_add_changelog.md | 8 - ...957_grier1_write_status_csv_immediately.md | 43 --- ...pitjain099_used_params_prefix_collision.md | 3 - ...149_jwhite242_hypothesis_test_stability.md | 4 - .../20260803_102412_jwhite242_submit_sleep.md | 3 - changelog.d/README.txt | 1 + docs/gen_contributing.py | 4 +- docs/gen_release.py | 30 ++ mkdocs.yml | 2 + pyproject.toml | 2 +- scripts/docs_prepare.py | 15 +- 14 files changed, 375 insertions(+), 66 deletions(-) create mode 100644 RELEASE.md delete mode 100644 changelog.d/20260408_182201_jwhite242_add_changelog.md delete mode 100644 changelog.d/20260720_084957_grier1_write_status_csv_immediately.md delete mode 100644 changelog.d/20260721_190000_arpitjain099_used_params_prefix_collision.md delete mode 100644 changelog.d/20260803_095149_jwhite242_hypothesis_test_stability.md delete mode 100644 changelog.d/20260803_102412_jwhite242_submit_sleep.md create mode 100644 changelog.d/README.txt create mode 100644 docs/gen_release.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c08c69e..ee914404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file. + +## 1.2.1 — 2026-08-10 + +### Changed + +- Write an initial `status.csv` as soon as the study graph has been constructed. Previously, the status file was not written until after scheduler submission of the first job step (when an asynchronous job step), or completion of the first job step (for a synchronous job step). Implemented in [PR #477](https://github.com/llnl/maestrowf/pull/477). + +### Fixed + +- `get_used_parameters` no longer matches a parameter whose name is a prefix of another, so a step using only `$(NP)` is no longer reported as also using `$(N)`. The token pattern now requires an exact name followed by an optional `.attribute`, and the name is escaped before it goes into the regex. Implemented in [PR #478](https://github.com/llnl/maestrowf/pull/478). + +- Guards sleep in submit loop to only trigger if a submission failure occurs. Significantly speeds up job submission rates which were previously capped at 1 per second due to this sleep. Implemented in [PR #481](https://github.com/llnl/maestrowf/pull/481). + +### Documentation + +- Add what's new section to publish changelog and additional roadmap related material. Implemented in [PR #472](https://github.com/llnl/maestrowf/pull/472). + +### Maintenance + +- Changelog automation via [scriv](https://github.com/nedbat/scriv). Implemented in [PR #472](https://github.com/llnl/maestrowf/pull/472) + +- Creates ci profile for hypothesis enabled tests to account for increased system variability via extending deadlines. Implemented in [PR #480](https://github.com/llnl/maestrowf/pull/480). + ## v1.2.0, 2026-03-27 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f71e4f26..21806b49 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ Hotfixes on `main` are allowed in principle, but no formal hotfix workflow has b ## Changelog fragments -This project uses `scriv` to manage changelog fragments. `scriv` is installed with the project dependencies, and creating the fragment is as simple as `scrive create` from the top level and then edit that fragment. Fragments are stored in: +This project uses `scriv` to manage changelog fragments. `scriv` is installed with the project dependencies, and creating the fragment is as simple as `scriv create` from the top level and then edit that fragment. Fragments are stored in: - `changelog.d/` @@ -117,7 +117,7 @@ The documentation includes a `What's New` section with: ## Release process -The release process is currently manual. +The release process is currently manual. See [RELEASE.md](RELEASE.md) for the maintainer checklist, TestPyPI/PyPI publishing commands, release-note guidance, and a proposed GitHub Actions automation plan. Typical steps: diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000..4e03a7c7 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,299 @@ +# Release Process + +This document describes the release process for `maestrowf`. + +The project uses a release-oriented branching model: + +- `develop` is the integration branch for normal development. +- `main` contains released code. +- release branches are cut from `develop` and merged back through the release flow. +- versioned release tags are created on `main`. + +## Branch and Merge Policy + +Use two merge policies: + +- ordinary feature, bugfix, documentation, and maintenance pull requests into `develop` may be squash merged +- release promotion and release mergeback pull requests should use merge commits + +Squash merging ordinary pull requests keeps `develop` readable, avoids requiring contributors to polish every intermediate review commit, and usually makes reverts simpler. The pull request, squash commit message, changelog fragment, and GitHub review history provide enough traceability for normal development. This may be revisited in the future with larger features/refactors. + +Release branches are promotion branches, so they should preserve ancestry. The release branch commit that was built, uploaded to TestPyPI, and manually validated should remain an ancestor of the official release. Use merge commits for: + +- `release/X.Y.Z` into `develop` +- `develop` into `main` +- `chore/mergeback_vX.Y.Z` into `develop` + +The next-development version bump branch, such as `chore/start-vNEXTdev0`, may be squash merged because it is a small mechanical change. We will keep it as a pull request rather than a direct push to develop so it remains visible, reviewable, and CI-gated. + +The release mergeback and next-development version bump are kept intentionally separate. The mergeback records release ancestry reconciliation. The version bump reopens development. Keeping them separate makes the history easier to audit. + +Avoid `dev` version tags unless a need arises for pre-release versions on pypi in the future to keep the git tag space less cluttered. Tags should normally identify final releases on `main`. Development version provenance is currently 1:1 with develop commits with the current squash merge approach; may revisit this if squash merges are abandoned in the future. + +## Release Goals + +A release should provide: + +- a clean version in `pyproject.toml` +- collected Scriv changelog fragments in `CHANGELOG.md` +- source distribution and wheel artifacts built from the release commit +- a TestPyPI upload suitable for manual installation and scheduler validation +- a final PyPI upload from the same reviewed release content +- a GitHub tag and release whose notes match the collected changelog entry +- release documentation built from the release commit + +Manual validation against real schedulers may remain outside GitHub Actions. This project does not have GitHub-hosted runners that can exercise site-specific Slurm, Flux, LSF, and machine policy behaviors and features. + +## Required Tools + +The repository currently pins Poetry for local development: + +```bash +pip install "poetry==1.8.4" +poetry install --with dev,docs +``` + +Scriv is available through the development dependencies on Python versions that support it. + +## Preparing a Release Branch + +Start from an up-to-date `develop` branch: + +```bash +git checkout develop +git pull --ff-only origin develop +git checkout -b release/X.Y.Z +``` + +Set the final release version: + +```bash +poetry version X.Y.Z +``` + +Collect the Scriv fragments into `CHANGELOG.md`: + +```bash +poetry run scriv collect +``` + +Check that: + +- `pyproject.toml` contains the final release version, not a `dev` version. +- `CHANGELOG.md` has a new `vX.Y.Z, YYYY-MM-DD` section. +- `changelog.d/` no longer contains the fragments that were collected for the release. +- the release notes are understandable without reading the merged pull requests. + +Run the local verification that is practical for the machine: + +```bash +poetry run pytest +poetry run python scripts/docs_prepare.py --mode release +poetry run mkdocs build +poetry build +``` + +Commit the release preparation: + +```bash +git add pyproject.toml CHANGELOG.md changelog.d +git commit -s -m "Finalize X.Y.Z release" +git push origin release/X.Y.Z +``` + +Do not merge the release branch yet. First publish this release branch commit to TestPyPI and validate it. + +## TestPyPI Publishing + +Publish to TestPyPI from the reviewed release branch or from the exact commit that will be merged for release. + +Clean any old local build artifacts first: + +```bash +rm -rf dist +poetry build +``` + +Configure Poetry to know about TestPyPI: + +```bash +poetry config repositories.testpypi https://test.pypi.org/legacy/ +``` + +Use an API token from TestPyPI. For local publishing, prefer an environment variable instead of storing the token in shell history: + +```bash +export TEST_PYPI_API_TOKEN="pypi-..." +poetry publish -r testpypi --username __token__ --password "$TEST_PYPI_API_TOKEN" +``` + +After the upload, test installation from TestPyPI in a clean environment: + +```bash +python -m venv /tmp/maestrowf-testpypi +. /tmp/maestrowf-testpypi/bin/activate +python -m pip install --upgrade pip +python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ maestrowf==X.Y.Z +maestro --version +``` + +The `--extra-index-url` is normally needed because TestPyPI does not mirror all runtime dependencies. + +Run the manual validation matrix that matters for the release. At minimum, record: + +- package version and wheel filename tested +- Python version +- operating system or cluster +- scheduler target tested, such as local, Slurm, Flux, or LSF +- representative study names or paths +- pass/fail result and any deviations + +If validation finds a release blocker, `develop`, update or add Scriv fragments as needed, and rebuild the release branch. Do not publish a different artifact with the same version number. + +### Validation records? + +Here's some ideas for potentially tracking release validation; these are currently only documented here, and not yet implemented. + +Record validation details in the release pull request or in a dedicated release validation issue: + +- release branch name +- release branch commit SHA +- workflow run URL or local build command +- artifact filenames and hashes +- TestPyPI project/version URL +- Python versions and platforms tested +- scheduler targets tested, such as local, Slurm, Flux, or LSF +- representative studies tested +- pass/fail result and any deviations + +GitHub Actions artifacts are useful during the release window, but they expire according to repository retention policy. We can not rely on them as the only long-term record. The durable long-term records should be the final Git tag, PyPI files, GitHub release assets and notes, and the release pull request or validation issue. + +## Merging to Main + +After TestPyPI validation passes: + +1. Open a pull request from `release/X.Y.Z` into `develop`. +2. Merge that pull request with a merge commit. +3. Open the release pull request from `develop` into `main`. +4. Verify CI on the `main` release pull request. +5. Merge the release pull request into `main` with a merge commit. +6. Tag the resulting `main` release commit: + +```bash +git checkout main +git pull --ff-only origin main +git tag -a vX.Y.Z -m "maestrowf vX.Y.Z" +git push origin vX.Y.Z +``` + +7. Publish the final package to PyPI from the tag. +8. Create `chore/mergeback_vX.Y.Z` from current `develop`, then merge `origin/main` into it: + +```bash +git checkout develop +git pull --ff-only origin develop +git checkout -b chore/mergeback_vX.Y.Z +git merge --no-ff origin/main -m "Mergeback of main into develop for vX.Y.Z" +git push origin chore/mergeback_vX.Y.Z +``` + +9. Open a pull request from `chore/mergeback_vX.Y.Z` into `develop`. +10. Merge that pull request with a merge commit. +11. Create `chore/start-vNEXTdev0` from updated `develop`: + +```bash +git checkout develop +git pull --ff-only origin develop +git checkout -b chore/start-vNEXTdev0 +poetry version NEXT_VERSIONdev0 +git add pyproject.toml +git commit -s -m "Start NEXT_VERSIONdev0 development" +git push origin chore/start-vNEXTdev0 +``` + +12. Open a pull request from `chore/start-vNEXTdev0` into `develop`. +13. Merge that pull request. Squash merge is acceptable for this version bump branch. + +Use the next appropriate version number for the development bump, for example `1.2.1dev0` after `1.2.0`. + +The `v1.2.0` release used this ancestry-preserving shape. Its mergeback was not a squash merge: + +- `68c1936` was the `main` release merge commit tagged `v1.2.0` +- `20f0daf` merged `main` into `chore/mergeback_v1.2.0` +- `78ffbe0` merged `chore/mergeback_v1.2.0` back into `develop` + +## Release Pull Request Checklist + +Use this checklist for the release pull request into `develop`: + +- final version set in `pyproject.toml` +- Scriv fragments collected into `CHANGELOG.md` +- release branch commit SHA recorded +- source distribution and wheel built +- TestPyPI upload completed +- clean install from TestPyPI verified +- manual scheduler validation recorded +- release notes reviewed +- CI passing + +Use this checklist for the release pull request into `main`: + +- source branch includes the validated release commit +- no unrelated changes entered `develop` after validation, or they were intentionally included and validated +- CI passing +- final PyPI publish will happen from the tag after merge + +Use this checklist for the mergeback pull request: + +- branch starts from current `develop` +- branch merges current `origin/main` +- no release-version or next-dev-version edit is combined into the mergeback +- pull request is merged with a merge commit + +Use this checklist for the next-development version bump pull request: + +- branch starts from `develop` after mergeback +- only the version bump is included +- squash merge is acceptable + +## PyPI Publishing + +Publish the final package from the tagged release commit. + +```bash +git checkout vX.Y.Z +rm -rf dist +poetry build +``` + +Use a PyPI API token: + +```bash +export PYPI_API_TOKEN="pypi-..." +poetry publish --username __token__ --password "$PYPI_API_TOKEN" +``` + +Once a version is published to PyPI, treat it as immutable. If a bad package is uploaded, release a new version. + +## GitHub Release Notes from Scriv + +`CHANGELOG.md` is the source of truth for release notes. Build the GitHub release notes from the Scriv-collected section for the tag. + +After `poetry run scriv collect`, inspect the generated section: + +```bash +sed -n '/^## vX.Y.Z, /,/^## v/p' CHANGELOG.md +``` + +For the GitHub release body, use the content under the `## vX.Y.Z, YYYY-MM-DD` heading and stop before the next `##` heading. Keep category headings such as `Added`, `Changed`, and `Fixed`. + +If using the GitHub CLI: + +```bash +awk '/^## vX.Y.Z, /{capture=1; next} capture && /^## v/{exit} capture{print}' CHANGELOG.md > /tmp/maestrowf-vX.Y.Z-notes.md +gh release create vX.Y.Z --title "maestrowf vX.Y.Z" --notes-file /tmp/maestrowf-vX.Y.Z-notes.md dist/* +``` + +This keeps the GitHub release, changelog, and documentation aligned. + +**NOTE:** May want/need to start with github's automatic draft release notes to get the text for linking the contributors; scriv doesn't currently record those in the changelog. diff --git a/changelog.d/20260408_182201_jwhite242_add_changelog.md b/changelog.d/20260408_182201_jwhite242_add_changelog.md deleted file mode 100644 index 0da5173d..00000000 --- a/changelog.d/20260408_182201_jwhite242_add_changelog.md +++ /dev/null @@ -1,8 +0,0 @@ -### Maintenance - -- Changelog automation via [scriv](https://github.com/nedbat/scriv). Implemented in [PR #472](https://github.com/llnl/maestrowf/pull/472) - - -### Documentation - -- Add what's new section to publish changelog and additional roadmap related material. Implemented in [PR #472](https://github.com/llnl/maestrowf/pull/472). diff --git a/changelog.d/20260720_084957_grier1_write_status_csv_immediately.md b/changelog.d/20260720_084957_grier1_write_status_csv_immediately.md deleted file mode 100644 index 12b04e11..00000000 --- a/changelog.d/20260720_084957_grier1_write_status_csv_immediately.md +++ /dev/null @@ -1,43 +0,0 @@ - - - - -### Changed - -- Write an initial `status.csv` as soon as the study graph has been constructed. Previously, the status file was not written until after scheduler submission of the first job step (when an asynchronous job step), or completion of the first job step (for a synchronous job step). Implemented in [PR #477](https://github.com/llnl/maestrowf/pull/477). - - - - - - diff --git a/changelog.d/20260721_190000_arpitjain099_used_params_prefix_collision.md b/changelog.d/20260721_190000_arpitjain099_used_params_prefix_collision.md deleted file mode 100644 index eef4be57..00000000 --- a/changelog.d/20260721_190000_arpitjain099_used_params_prefix_collision.md +++ /dev/null @@ -1,3 +0,0 @@ -### Fixed - -- `get_used_parameters` no longer matches a parameter whose name is a prefix of another, so a step using only `$(NP)` is no longer reported as also using `$(N)`. The token pattern now requires an exact name followed by an optional `.attribute`, and the name is escaped before it goes into the regex. Implemented in [PR #478](https://github.com/llnl/maestrowf/pull/478). diff --git a/changelog.d/20260803_095149_jwhite242_hypothesis_test_stability.md b/changelog.d/20260803_095149_jwhite242_hypothesis_test_stability.md deleted file mode 100644 index d8a3aa83..00000000 --- a/changelog.d/20260803_095149_jwhite242_hypothesis_test_stability.md +++ /dev/null @@ -1,4 +0,0 @@ -### Maintenance - -- Creates ci profile for hypothesis enabled tests to account for increased system variability via extending deadlines. Implemented in [PR #480](https://github.com/llnl/maestrowf/pull/480). - diff --git a/changelog.d/20260803_102412_jwhite242_submit_sleep.md b/changelog.d/20260803_102412_jwhite242_submit_sleep.md deleted file mode 100644 index 0409aecf..00000000 --- a/changelog.d/20260803_102412_jwhite242_submit_sleep.md +++ /dev/null @@ -1,3 +0,0 @@ -### Fixed - -- Guards sleep in submit loop to only trigger if a submission failure occurs. Significantly speeds up job submission rates which were previously capped at 1 per second due to this sleep. Implemented in [PR #481](https://github.com/llnl/maestrowf/pull/481). diff --git a/changelog.d/README.txt b/changelog.d/README.txt new file mode 100644 index 00000000..22ca29c7 --- /dev/null +++ b/changelog.d/README.txt @@ -0,0 +1 @@ +This directory will hold the changelog entries managed by scriv. diff --git a/docs/gen_contributing.py b/docs/gen_contributing.py index a1d18b3f..2fcec158 100644 --- a/docs/gen_contributing.py +++ b/docs/gen_contributing.py @@ -26,7 +26,9 @@ def get_contributing_content() -> str: Markdown content for the generated contributing page. """ if SOURCE.exists(): - return SOURCE.read_text(encoding="utf-8") + return SOURCE.read_text(encoding="utf-8").replace( + "(RELEASE.md)", "(release.md)" + ) return "# Contributing\n\nContributor guide not found.\n" diff --git a/docs/gen_release.py b/docs/gen_release.py new file mode 100644 index 00000000..18fe4f67 --- /dev/null +++ b/docs/gen_release.py @@ -0,0 +1,30 @@ +"""Generate the release-process page for the MkDocs site. + +This script writes ``whats_new/release.md`` using ``RELEASE.md`` from the +repository root when it is available. If the source file does not exist, it +writes a placeholder page instead. +""" + +from pathlib import Path + +import mkdocs_gen_files + + +REPO_ROOT = Path(__file__).resolve().parent.parent +SOURCE = REPO_ROOT / "RELEASE.md" +OUTPUT_PATH = "whats_new/release.md" + + +def get_release_content() -> str: + """Return release-process content for documentation generation.""" + if SOURCE.exists(): + return SOURCE.read_text(encoding="utf-8") + + return "# Release Process\n\nRelease process guide not found.\n" + + +with mkdocs_gen_files.open(OUTPUT_PATH, "w") as fd: + fd.write(get_release_content()) + +if SOURCE.exists(): + mkdocs_gen_files.set_edit_path(OUTPUT_PATH, SOURCE) diff --git a/mkdocs.yml b/mkdocs.yml index f9d1860e..5061af29 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,6 +36,7 @@ plugins: - docs/gen_api_docs.py - docs/gen_changelog.py - docs/gen_contributing.py + - docs/gen_release.py - literate-nav: nav_file: SUMMARY.md - glightbox @@ -110,6 +111,7 @@ nav: - Overview: whats_new/index.md - Changelog: whats_new/changelog.md - Contributing: whats_new/contributing.md + - Release Process: whats_new/release.md extra: social: diff --git a/pyproject.toml b/pyproject.toml index f2075eab..7539b7ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool] [tool.poetry] name = "maestrowf" -version = "1.2.1dev5" +version = "1.2.1" description = "A tool to easily orchestrate general computational workflows both locally and on supercomputers." license = "MIT License" classifiers = [ diff --git a/scripts/docs_prepare.py b/scripts/docs_prepare.py index 7a466b08..aed0a269 100644 --- a/scripts/docs_prepare.py +++ b/scripts/docs_prepare.py @@ -33,6 +33,11 @@ def run(cmd: list[str], cwd: Path) -> None: subprocess.run(cmd, cwd=cwd, check=True) +def has_changelog_fragments(fragment_dir: Path) -> bool: + """Return whether Scriv has Markdown fragments to collect.""" + return fragment_dir.exists() and any(fragment_dir.glob("*.md")) + + def prepare_docs_changelog(repo_root: Path, mode: str, title: str | None = None) -> None: """Create the changelog file used by the documentation build. @@ -55,8 +60,9 @@ def prepare_docs_changelog(repo_root: Path, mode: str, title: str | None = None) """ changelog = repo_root / "CHANGELOG.md" docs_changelog = repo_root / "CHANGELOG_docs.md" - # TODO: Consider moving this into scripts too to avoid repo-root clutter? + # TODO: Consider moving this into scripts too to avoid repo-root clutter? scriv_config = repo_root / "scriv_docs.ini" + fragment_dir = repo_root / "changelog.d" if not changelog.exists(): raise FileNotFoundError(f"Missing source changelog: {changelog}") @@ -65,6 +71,13 @@ def prepare_docs_changelog(repo_root: Path, mode: str, title: str | None = None) shutil.copyfile(changelog, docs_changelog) if mode == "dev": + if not has_changelog_fragments(fragment_dir): + print( + f"No changelog fragments found in {fragment_dir}; " + "skipping scriv collect." + ) + return + cmd = [ "scriv", "collect",