feat: modernize Python tooling (pyproject.toml + uv + semantic-release)

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,13 +8,8 @@ Add some.
 **Merge checklist:**
 - [ ] All reviewers approved
 - [ ] CI build is green
-- [ ] Version bumped
-- [ ] Changelog record added
 - [ ] Documentation updated (not only docstrings)
 - [ ] Commits are squashed
 
 **Post merge:**
-- [ ] Create a tag
-- [ ] Check new version is pushed to PyPI after tag-triggered build is 
-      finished.
 - [ ] Delete working branch (if not needed anymore)

.github/workflows/ci.yml

@@ -6,6 +6,8 @@ on:
   pull_request:
     branches:
     - '**'
+  # So the release workflow can run CI before cutting a release.
+  workflow_call:
 
 
 jobs:
@@ -21,25 +23,24 @@ jobs:
 
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-    - name: setup python
-      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
       with:
+        enable-cache: true
         python-version: ${{ matrix.python-version }}
 
-    - name: Install pip
-      run: pip install -r requirements/pip.txt
-
     - name: Install Dependencies
-      run: pip install -r requirements/ci.txt
+      run: uv sync --group ci
 
     - name: Run Tests
       env:
         TOXENV: ${{ matrix.toxenv }}
-      run: tox
+      run: uv run tox
 
     - name: Run coverage
       if: matrix.python-version == '3.12' && matrix.toxenv == 'django52'
-      uses: codecov/codecov-action@v7
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: unittests

.github/workflows/release.yml

@@ -0,0 +1,103 @@
+name: Semantic Release
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  run_ci:
+    uses: ./.github/workflows/ci.yml
+
+  release:
+    needs: run_ci
+    runs-on: ubuntu-latest
+    if: github.ref_name == 'main'
+    concurrency:
+      group: ${{ github.workflow }}-release-${{ github.ref_name }}
+      cancel-in-progress: false
+
+    permissions:
+      contents: write
+
+    steps:
+      # Note: We checkout the repository at the branch that triggered the workflow.
+      # Python Semantic Release will automatically convert shallow clones to full clones
+      # if needed to ensure proper history evaluation. However, we forcefully reset the
+      # branch to the workflow sha because it is possible that the branch was updated
+      # while the workflow was running, which prevents accidentally releasing un-evaluated
+      # changes.
+      - name: Setup | Checkout Repository on Release Branch
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: Setup | Force release branch to be at workflow sha
+        run: |
+          git reset --hard ${{ github.sha }}
+
+      - name: Action | Semantic Version Release
+        id: release
+        uses: python-semantic-release/python-semantic-release@9a026e9303981c866c3425723009becb2437c757 # v10.6.2
+        with:
+          github_token: ${{ secrets.OPENEDX_SEMANTIC_RELEASE_GITHUB_TOKEN }}
+          git_committer_name: "github-actions"
+          git_committer_email: "actions@users.noreply.github.com"
+          changelog: "false"
+          # Commit, tag, push and build, but don't create the GitHub release.
+          # We create it ourselves in the next step so that the distributions
+          # are attached before the release is published. See that step for why.
+          vcs_release: "false"
+
+      # This repo has immutable releases enabled, which freezes a release's
+      # assets the moment it is published, so assets cannot be attached
+      # afterwards. `gh release create` handles this by creating the release as
+      # a draft, uploading the assets, and only then publishing it:
+      # https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/immutable-releases
+      - name: Publish | Create GitHub Release with Assets
+        if: steps.release.outputs.released == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_NOTES: ${{ steps.release.outputs.release_notes }}
+          TAG: ${{ steps.release.outputs.tag }}
+        run: |
+          printf '%s' "$RELEASE_NOTES" > "$RUNNER_TEMP/release_notes.md"
+          gh release create "$TAG" \
+            --verify-tag \
+            --title "$TAG" \
+            --notes-file "$RUNNER_TEMP/release_notes.md" \
+            dist/*
+
+      - name: Upload | Distribution Artifacts
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: steps.release.outputs.released == 'true'
+        with:
+          name: distribution-artifacts
+          path: dist
+          if-no-files-found: error
+
+    outputs:
+      released: ${{ steps.release.outputs.released || 'false' }}
+      version: ${{ steps.release.outputs.version }}
+
+  publish_to_pypi:
+    # 1. Separate out the publish step from the github release step to run each step at
+    #    the least amount of token privilege
+    # 2. Also, publishing can fail, and its better to have a separate job if you need to retry
+    #    and it won't require reversing the release.
+    runs-on: ubuntu-latest
+    needs: release
+    if: github.ref_name == 'main' && needs.release.outputs.released == 'true'
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Setup | Download Build Artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: distribution-artifacts
+          path: dist
+
+      - name: Publish to PyPi
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

.gitignore

@@ -33,6 +33,10 @@ pip-log.txt
 coverage.xml
 htmlcov/
 
+# Virtual environments
+venv/
+.venv/
+
 .vscode/
 
 # The Silver Searcher

.readthedocs.yaml

@@ -9,7 +9,11 @@ version: 2
 sphinx:
   configuration: docs/conf.py
 
-python:
-  version: 3.8
-  install:
-    - requirements: requirements/doc.txt
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.12"
+  jobs:
+    install:
+      - pip install uv
+      - uv sync --group doc

CHANGELOG.rst

@@ -15,11 +15,6 @@ Unreleased
 **********
 * Nothing unreleased
 
-[2.0.0]
-********
-* chore: Upgrade Python requirements
-
-
 [1.8.0]
 ********
 * chore: Upgrade Python requirements

MANIFEST.in

@@ -1,6 +1,3 @@
-include CHANGELOG.rst
 include LICENSE.txt
 include README.rst
-include requirements/base.in
-include requirements/constraints.txt
-recursive-include openedx_ledger *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg
+recursive-include src/openedx_ledger *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean compile_translations coverage diff_cover docs dummy_translations \
+.PHONY: clean compile-requirements compile_translations coverage diff_cover docs dummy_translations \
         extract_translations fake_translations help pii_check pull_translations push_translations \
         quality requirements selfcheck test test-all upgrade validate install_transifex_client
 
@@ -26,66 +26,60 @@ coverage: clean ## generate and view HTML coverage report
 	$(BROWSER)htmlcov/index.html
 
 docs: ## generate Sphinx HTML documentation, including API docs
-	tox -e docs
+	uv sync --group doc
+	DJANGO_SETTINGS_MODULE=test_settings PYTHONPATH=$(CURDIR) SPHINXOPTS=-W uv run doc8 --ignore-path docs/_build README.rst docs
+	rm -f docs/openedx_ledger.rst
+	rm -f docs/modules.rst
+	DJANGO_SETTINGS_MODULE=test_settings PYTHONPATH=$(CURDIR) SPHINXOPTS=-W uv run make -e -C docs clean
+	DJANGO_SETTINGS_MODULE=test_settings PYTHONPATH=$(CURDIR) SPHINXOPTS=-W uv run make -e -C docs html
+	uv run python -m build --wheel
+	uv run twine check dist/*
 	$(BROWSER)docs/_build/html/index.html
 
-# Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
-PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
-
-upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	pip install -qr requirements/pip-tools.txt
-	# Make sure to compile files after any other files they include!
-	$(PIP_COMPILE) --allow-unsafe -o requirements/pip.txt requirements/pip.in
-	$(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip install -qr requirements/pip.txt
-	pip install -qr requirements/pip-tools.txt
-	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
-	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
-	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
-	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
-	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
-	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
-	# Let tox control the Django version for tests
-	sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp
-	mv requirements/test.tmp requirements/test.txt
+compile-requirements: ## generate the uv.lock file without upgrading packages
+	uv lock
+
+upgrade: ## upgrade all packages in uv.lock and sync constraints from edx-lint
+	uv run --with edx-lint edx_lint write_uv_constraints pyproject.toml
+	uv lock --upgrade
 
 quality: ## check coding style with pycodestyle and pylint
-	tox -e quality
+	uv sync --group quality
+	touch tests/__init__.py
+	uv run pylint src/openedx_ledger tests manage.py
+	rm tests/__init__.py
+	uv run pycodestyle src/openedx_ledger tests manage.py
+	uv run isort --check-only --diff tests src/openedx_ledger manage.py test_settings.py
+	$(MAKE) selfcheck
 
 pii_check: ## check for PII annotations on all Django models
-	tox -e pii_check
-
-piptools: ## install pinned version of pip-compile and pip-sync
-	pip install -r requirements/pip.txt
-	pip install -r requirements/pip-tools.txt
+	DJANGO_SETTINGS_MODULE=test_settings uv run code_annotations django_find_annotations --config_file .pii_annotations.yml --lint --report --coverage
 
-requirements: piptools ## install development environment requirements
-	pip-sync -q requirements/dev.txt requirements/private.*
+requirements: ## install development environment requirements
+	uv sync --group dev
 
 test: clean ## run tests in the current virtualenv
 	DJANGO_SETTINGS_MODULE=test_settings pytest
 
 diff_cover: test ## find diff lines that need test coverage
 	diff-cover coverage.xml
 
-test-all: quality pii_check ## run tests on every supported Python/Django combination
-	tox
-	tox -e docs
+test-all: quality pii_check docs ## run tests on every supported Python/Django combination
+	uv run tox
 
 validate: quality pii_check test ## run tests and quality checks
 
 selfcheck: ## check that the Makefile is well-formed
 	@echo "The Makefile is well-formed."
 
 isort:
-	isort tests openedx_ledger manage.py setup.py test_settings.py
+	isort tests src/openedx_ledger manage.py test_settings.py
 
 style:
-	pycodestyle openedx_ledger tests manage.py setup.py
+	pycodestyle src/openedx_ledger tests manage.py
 
 lint:
-	pylint openedx_ledger tests manage.py setup.py
+	pylint src/openedx_ledger tests manage.py
 
 ## Docker in this repo is only supported for running tests locally
 ## as an alternative to virtualenv natively
@@ -96,14 +90,14 @@ test-shell: ## Run a shell, as root, on the specified service container
 
 extract_translations: ## extract strings to be translated, outputting .mo files
 	rm -rf docs/_build
-	cd openedx_ledger && ../manage.py makemessages -l en -v1 -d django
-	cd openedx_ledger && ../manage.py makemessages -l en -v1 -d djangojs
+	cd src/openedx_ledger && ../../manage.py makemessages -l en -v1 -d django
+	cd src/openedx_ledger && ../../manage.py makemessages -l en -v1 -d djangojs
 
 compile_translations: ## compile translation files, outputting .po files for each supported language
-	cd openedx_ledger && ../manage.py compilemessages
+	cd src/openedx_ledger && ../../manage.py compilemessages
 
 detect_changed_source_translations:
-	cd openedx_ledger && i18n_tool changed
+	cd src/openedx_ledger && i18n_tool changed
 
 pull_translations: ## pull translations from Transifex
 	tx pull -a -f -t --mode reviewed
@@ -112,7 +106,7 @@ push_translations: ## push source translation files (.po) from Transifex
 	tx push -s
 
 dummy_translations: ## generate dummy translation (.po) files
-	cd openedx_ledger && i18n_tool dummy
+	cd src/openedx_ledger && i18n_tool dummy
 
 build_dummy_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files
 

README.rst

@@ -47,12 +47,11 @@ Every time you develop something in this repo
    # inside the container shell now
    # TODO find a better permanent place to put this git config call
    git config --global --add safe.directory /edx/src/openedx-ledger
-   pip install -e /edx/src/openedx-ledger
-   pip freeze | grep ledger # should give output pointing at a file and git hash
+   pushd /edx/src/openedx-ledger
+   make requirements # installs dependencies via uv into this repo's own .venv
+   uv pip show openedx-ledger # should give output pointing at a file and git hash
 
    # Now, to run tests and validation against openedx-ledger:
-   pushd /edx/src/openedx-ledger
-   source venv/bin/activate
    make validate