feat: modernize Python tooling (pyproject.toml + uv + semantic-release)

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,16 +10,9 @@
       - It may or may not make a migration depending on exactly what you modified, but it should still be run.
     - This should be run from either a venv with all the lms/edx-enterprise requirements installed or if you checked out edx-enterprise into the src directory used by lms, you can run this command through an lms shell.
         - It would be `./manage.py lms makemigrations` in the shell.
-- [ ] [Version](https://github.com/openedx/edx-enterprise/blob/master/enterprise/__init__.py) bumped
-- [ ] [Changelog](https://github.com/openedx/edx-enterprise/blob/master/CHANGELOG.rst) record added
 - [ ] Translations updated (see docs/internationalization.rst but also this isn't blocking for merge atm)
 
 **Post merge:**
-- [ ] Tag pushed and a new [version](https://github.com/openedx/edx-enterprise/releases) released
-    - *Note*: Assets will be added automatically. You just need to provide a tag (should match your version number) and title and description.
-- [ ] After versioned build finishes in [GitHub Actions](https://github.com/openedx/edx-enterprise/actions), verify version has been pushed to [PyPI](https://pypi.org/project/edx-enterprise/)
-    - Each step in the release build has a condition flag that checks if the rest of the steps are done and if so will deploy to PyPi.
-    (so basically once your build finishes, after maybe a minute you should see the new version in PyPi automatically (on refresh))
 - [ ] PR created in [edx-platform](https://github.com/openedx/edx-platform) to upgrade dependencies (including edx-enterprise)
     - Trigger the '[Upgrade one Python dependency](https://github.com/openedx/edx-platform/actions/workflows/upgrade-one-python-dependency.yml)' action against master in edx-platform with new version number to generate version bump PR
     - This **must** be done after the version is visible in PyPi as `make upgrade` in edx-platform will look for the latest version in PyPi.

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches: [master]
   pull_request:
+  workflow_call:
 
 concurrency:
   group: ci-${{ github.event.pull_request.number || github.ref }}
@@ -21,19 +22,20 @@ jobs:
       RUNJSHINT: true
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-    - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-      with:
-        python-version: ${{ matrix.python-version }}
-        architecture: x64
     - name: Setup Nodejs
       uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
       with:
         node-version-file: '.nvmrc'
-    - run: pip install -r requirements/ci.txt
+    - name: Install uv
+      uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+      with:
+        enable-cache: true
+        python-version: ${{ matrix.python-version }}
+    - run: uv sync --group ci
     - name: Run tox
       env:
         TOXENV: ${{ matrix.toxenv }}
-      run: tox
+      run: uv run tox
     - name: Run code coverage
       if: matrix.python-version == '3.12' && matrix.toxenv == 'django52' # Only run this once as part of tests
       uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1

.github/workflows/mysql8-migrations.yml

@@ -33,31 +33,18 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y libxmlsec1-dev
-    - name: Get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-    - name: Cache pip dependencies
-      id: cache-dependencies
-      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+    - name: Install uv
+      uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
       with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/pip_tools.txt') }}
-        restore-keys: ${{ runner.os }}-pip-
+        enable-cache: true
 
     - name: Ubuntu and sql Versions
       run: |
         lsb_release -a
         mysql -V
     - name: Install Python dependencies
       run: |
-        pip install -qr requirements/dev.txt --exists-action w
-        pip-sync requirements/test-master.txt requirements/dev.txt requirements/private.* requirements/test.txt
-        pip uninstall -y mysqlclient
-        pip install --no-binary mysqlclient mysqlclient
-        pip uninstall -y xmlsec
-        pip install --no-binary xmlsec xmlsec
-        pip install backports.zoneinfo
+        uv sync --group dev --no-binary-package mysqlclient --no-binary-package xmlsec
     - name: Initiate Services
       run: |
         sudo /etc/init.d/mysql start
@@ -78,5 +65,5 @@ jobs:
       run: |
         echo "CREATE DATABASE IF NOT EXISTS enterprise;" | sudo mysql -u root
         echo "Running the migrations."
-        python manage.py migrate --settings=enterprise.settings.test
+        uv run python manage.py migrate --settings=enterprise.settings.test
         echo "use mysql; select * from user;" | sudo mysql -u root

.github/workflows/postgresql-migrations.yml

@@ -42,11 +42,15 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libxmlsec1-dev
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: true
+
       - name: Install Python dependencies
         run: |
-          pip3 install --upgrade pip
-          pip3 install -r requirements/dev.txt
-          pip3 install psycopg2-binary
+          uv sync --group dev --group postgresql
 
       - name: Run migrations
         env:
@@ -58,5 +62,5 @@ jobs:
           DB_PORT: 5432
         run: |
           echo "Running the migrations."
-          python3 manage.py migrate --settings=enterprise.settings.test
+          uv run python manage.py migrate --settings=enterprise.settings.test
 

.github/workflows/release.yml

@@ -0,0 +1,117 @@
+name: Semantic Release
+
+on:
+  push:
+    branches: [master]
+
+jobs:
+  # The same aggregate a pull request has to pass, so the release runs exactly
+  # the checks that were required to merge -- and picks up any package added to
+  # ci.yml later without needing a change here.
+  run_ci:
+    uses: ./.github/workflows/ci.yml
+
+  release:
+    # Every package this workflow publishes has to build before we tag
+    # anything. The GitHub release and the PyPI uploads cannot be taken back,
+    # so a package that only fails to build in its publish job would leave the
+    # release half-finished.
+    needs: [run_ci]
+    runs-on: ubuntu-latest
+    if: github.ref_name == 'master'
+    concurrency:
+      group: ${{ github.workflow }}-release-${{ github.ref_name }}
+      cancel-in-progress: false
+
+    permissions:
+      contents: write
+
+    steps:
+      # Note: We checkout the repository at the branch that triggered the workflow.
+      # Python Semantic Release will automatically convert shallow clones to full clones
+      # if needed to ensure proper history evaluation. However, we forcefully reset the
+      # branch to the workflow sha because it is possible that the branch was updated
+      # while the workflow was running, which prevents accidentally releasing un-evaluated
+      # changes.
+      - name: Setup | Checkout Repository on Release Branch
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: Setup | Force release branch to be at workflow sha
+        run: |
+          git reset --hard ${{ github.sha }}
+
+      - name: Action | Semantic Version Release
+        id: release
+        # Adjust tag with desired version if applicable.
+        uses: python-semantic-release/python-semantic-release@b700dbeb1f431a0fcc30a79f3f0869407fe07278 # v10.7.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          git_committer_name: "github-actions"
+          git_committer_email: "actions@users.noreply.github.com"
+          changelog: "false"
+          # Commit, tag, push and build, but don't create the GitHub release.
+          # We create it ourselves in the next step so that the distributions
+          # are attached before the release is published. See that step for why.
+          vcs_release: "false"
+
+      # This repo has immutable releases enabled, which freezes a release's
+      # assets the moment it is published, so assets cannot be attached
+      # afterwards. Everything we want on the release has to be built by now
+      # and passed to this one command, which creates the release as a draft,
+      # uploads the assets, and only then publishes it:
+      # https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/immutable-releases
+      - name: Publish | Create GitHub Release with Assets
+        if: steps.release.outputs.released == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Reuse the release notes python-semantic-release generated for us.
+          RELEASE_NOTES: ${{ steps.release.outputs.release_notes }}
+          TAG: ${{ steps.release.outputs.tag }}
+        run: |
+          # Output the release notes to a file
+          printf '%s' "$RELEASE_NOTES" > "$RUNNER_TEMP/release_notes.md"
+          # Creates the release as a draft, uploads the assets, and then
+          # publishes it -- all within this one command.
+          gh release create "$TAG" \
+            --verify-tag \
+            --title "$TAG" \
+            --notes-file "$RUNNER_TEMP/release_notes.md" \
+            dist/*
+
+      - name: Upload | Distribution Artifacts
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: steps.release.outputs.released == 'true'
+        with:
+          name: distribution-artifacts
+          path: dist
+          if-no-files-found: error
+
+    outputs:
+      released: ${{ steps.release.outputs.released || 'false' }}
+      version: ${{ steps.release.outputs.version }}
+
+  publish_to_pypi:
+    # 1. Separate out the publish step from the github release step to run each step at
+    #    the least amount of token privilege
+    # 2. Also, publishing can fail, and its better to have a separate job if you need to retry
+    #    and it won't require reversing the release.
+    runs-on: ubuntu-latest
+    needs: release
+    if: github.ref_name == 'master' && needs.release.outputs.released == 'true'
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Setup | Download Build Artifacts
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        id: artifact-download
+        with:
+          name: distribution-artifacts
+          path: dist
+
+      - name: Publish to PyPi
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

.gitignore

@@ -96,6 +96,7 @@ enterprise/static/enterprise/bundles/*.js
 
 # Virtual environments
 venv/
+.venv/
 
 # pyenv
 .python-version

.readthedocs.yaml

@@ -6,9 +6,9 @@
 version: 2
 
 build:
-   os: "ubuntu-lts-latest"
-   tools:
-      python: "3.12"
+  os: ubuntu-26.04
+  tools:
+    python: "3.12"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
@@ -20,7 +20,10 @@ formats:
   - pdf
   - epub
 
-# Optionally set the version of Python and requirements required to build your docs
+# Install dependencies via uv, using the "doc" dependency group from pyproject.toml
 python:
   install:
-  - requirements: requirements/doc.txt
+  - method: uv
+    command: sync
+    groups:
+    - doc

Dockerfile

@@ -63,14 +63,19 @@ WORKDIR /edx/app/edx-enterprise
 
 RUN python3.12 -m venv $VIRTUAL_ENV
 
-# Copy the requirements explicitly even though we copy everything below
+# uv installs into $UV_PROJECT_ENVIRONMENT instead of its default .venv, so it
+# reuses the venv created above (kept on PATH by the ENV lines further up).
+ENV UV_PROJECT_ENVIRONMENT="${VIRTUAL_ENV}"
+
+# Copy the dependency manifests explicitly even though we copy everything below
 # this prevents the image cache from busting unless the dependencies have changed.
-COPY requirements/ /edx/app/edx-enterprise/requirements/
+COPY pyproject.toml uv.lock /edx/app/edx-enterprise/
 COPY package.json /edx/app/edx-enterprise/package.json
 COPY package-lock.json /edx/app/edx-enterprise/package-lock.json
 
 # Fetch and install dependencies.
-RUN pip install -r requirements/dev.txt
+RUN pip install uv
+RUN uv sync --group dev --frozen
 RUN pip install nodeenv
 
 # Set up a Node environment and install Node requirements.

MANIFEST.in

@@ -1,11 +1,9 @@
 include AUTHORS
-include CHANGELOG.rst
 include CONTRIBUTING.rst
 include LICENSE.txt
 include README.rst
-recursive-include enterprise *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt *.po *.mo
-recursive-include enterprise_learner_portal *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt
-recursive-include integrated_channels *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt
-recursive-include consent *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt
-recursive-include requirements *.txt *.in
-include requirements/constraints.txt
+recursive-include src/enterprise *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt *.po *.mo
+exclude src/enterprise/locale
+recursive-include src/enterprise_learner_portal *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt
+recursive-include src/integrated_channels *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt
+recursive-include src/consent *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.txt