feat: using pytest as a test runner, diversify the tests into unit, integration, and performance tests (#1028)

Author: jenstroeger

.pre-commit-config.yaml

@@ -87,7 +87,7 @@ repos:
     language: python
     files: ^src/package/|^tests/
     types: [text, python]
-    args: [--config-file, pyproject.toml]
+    args: [--explicit-package-bases, --config-file, pyproject.toml]
 
 # Check for potential security issues.
 - repo: https://github.com/PyCQA/bandit
@@ -160,14 +160,14 @@ repos:
   hooks:
   - id: actionlint
 
-# On push to the remote, run the unit tests. Note that the `COVERAGE_CORE` variable is
-# required for Python 3.12+ to make sure Coverage uses the new Python monitoring module.
+# On push to the remote, run all tests. Note that the `COVERAGE_CORE` variable is required
+# for Python 3.12+ to make sure Coverage uses the new Python monitoring module.
 # See also: https://blog.trailofbits.com/2025/05/01/making-pypis-test-suite-81-faster/#optimizing-coverage-with-python-312s-sysmonitoring
 - repo: local
   hooks:
   - id: pytest
     name: Run unit tests
-    entry: env COVERAGE_CORE=sysmon pytest -c pyproject.toml --cov-config pyproject.toml src/package/ tests/ docs/
+    entry: env COVERAGE_CORE=sysmon pytest --config-file pyproject.toml --cov-config pyproject.toml -m 'not integration and not performance' src/package/ tests/ docs/
     language: python
     verbose: true
     always_run: true

Makefile

@@ -165,20 +165,27 @@ check-actionlint:
 check:
 	pre-commit run --all-files
 
-# Run all unit tests. The --files option avoids stashing but passes files; however,
-# the hook setup itself does not pass files to pytest (see .pre-commit-config.yaml).
-.PHONY: test
-test:
-	pre-commit run pytest --hook-stage push --files tests/
+# Run different kinds of tests: unit tests, integration tests, performance tests.
+# Note that the default goal 'test' runs the unit tests only, mainly for convenience
+# and compatibility with existing scripts.
+.PHONY: test test-all test-unit test-integration test-performance
+test: test-unit
+test-unit:
+	COVERAGE_CORE=sysmon python -m pytest --config-file pyproject.toml --cov-config pyproject.toml -m 'not integration and not performance' src/package/ tests/ docs/
+test-integration:
+	python -m pytest --config-file pyproject.toml --no-cov -m integration tests/
+test-performance:
+	python -m pytest --config-file pyproject.toml --no-cov -m performance tests/
+test-all: test-unit test-integration test-performance
 
 # Build a source distribution package and a binary wheel distribution artifact.
 # When building these artifacts, we need the environment variable SOURCE_DATE_EPOCH
 # set to the build date/epoch. For more details, see: https://flit.pypa.io/en/latest/reproducible.html
 .PHONY: dist
 dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt
-dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: check test dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt
+dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: check test-all dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt
 	SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format wheel
-dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: check test dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt
+dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: check test-all dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt
 	SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format sdist
 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip: docs-html
 	python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip docs/_build/html/

README.md

@@ -65,6 +65,7 @@ test = [
     "faker ==38.2.0",
     "hypothesis >=6.21.0,<6.148.6",
     "pytest >=7.2.0,<9.0.0",
+    "pytest-benchmark ==5.2.0",
     "pytest-cases ==3.9.1",
     "pytest-custom_exit_code ==0.3.0",
     "pytest-cov ==6.3.0",  # Uses: coverage[toml] >=7.5
@@ -258,7 +259,7 @@ max-line-length = 120
 # https://github.com/yashtodi94/pytest-custom_exit_code
 [tool.pytest.ini_options]
 minversion = "7.0"
-addopts = """-vv -ra --tb native --durations 0 \
+addopts = """-vv -ra --tb native --durations 0 --strict-markers --import-mode importlib \
     --hypothesis-show-statistics --hypothesis-explain --hypothesis-verbosity verbose \
     --doctest-modules --doctest-continue-on-failure --doctest-glob '*.rst' --doctest-plus \
     --suppress-no-test-exit-code \
@@ -281,3 +282,7 @@ filterwarnings = [
     "error::pytest.PytestUnraisableExceptionWarning",
     "error::pytest.PytestUnhandledThreadExceptionWarning",
 ]
+markers = [
+    "integration: more complex application-level integration tests.",
+    "performance: performance tests.",
+]

pyproject.toml

@@ -0,0 +1,16 @@
+"""Test the Package itself using its external interface as in integration into a larger run context."""
+
+# https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b404-import-subprocess
+import subprocess  # nosec B404
+
+import pytest
+
+
+@pytest.mark.integration
+def test_package() -> None:
+    """Test the Something command."""
+    # For testing we disable this warning here:
+    # https://bandit.readthedocs.io/en/latest/plugins/b603_subprocess_without_shell_equals_true.html
+    # https://bandit.readthedocs.io/en/latest/plugins/b607_start_process_with_partial_path.html
+    completed = subprocess.run(["something"], check=True, shell=False)  # nosec B603, B607
+    assert completed.returncode == 0

tests/integration/test_something.py

@@ -0,0 +1,12 @@
+"""Test the performance of various package parts, or the package as a whole."""
+
+import pytest
+from pytest_benchmark.fixture import BenchmarkFixture
+
+from package.something import Something
+
+
+@pytest.mark.performance
+def test_something(benchmark: BenchmarkFixture) -> None:
+    """Test performance of the function."""
+    benchmark.pedantic(Something.do_something, iterations=10, rounds=100)  # type: ignore[no-untyped-call]

tests/performance/test_something.py

@@ -1,4 +1,4 @@
-"""Test the Something module. Add more tests here, as needed."""
+"""Test the Something module as a unit test. Add more tests here, as needed."""
 
 import faker
 from hypothesis import given, strategies