From 0c8917c961de25dca94ab7e9eecfd45b4bca001f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 20 Oct 2026 13:08:08 +0200 Subject: [PATCH] Add more details to THIRD_PARTY.md Remove license comments from pyproject.toml --- DEVELOPER_GUIDE.md | 18 +++++++-- Makefile | 17 ++++++-- THIRD_PARTY.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 39 +----------------- 4 files changed, 130 insertions(+), 43 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 7109a421..8315ec14 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -136,9 +136,8 @@ Always commit both `pyproject.toml` and `uv.lock` together. ### Third-party license attribution -`THIRD_PARTY.md` lists the runtime dependencies with their licenses, authors and URLs. Regenerate -it whenever the dependencies change, and before a release, so the `NOTICE` attribution can be -checked against it: +`THIRD_PARTY.md` inventories the runtime dependencies with their licenses, authors and URLs. +Regenerate it whenever the dependencies change: ```bash make licenses @@ -148,6 +147,19 @@ The target runs [pip-licenses](https://github.com/raimon49/pip-licenses) in an i environment that contains only the runtime dependencies, so the develop and test extras are not listed. Commit the updated `THIRD_PARTY.md` together with `pyproject.toml` and `uv.lock`. +Only the table below the `GENERATED-TABLE-BELOW` marker is replaced. Everything above it is +maintained by hand and records what the generated table cannot: corrections where a package +declares only a coarse classifier, the elections made for dual-licensed packages, the Category B +dependencies, and which dependencies ship a `NOTICE` of their own. **When a dependency is added, +removed or upgraded, review those sections too** — a new dependency may need a correction or an +election, and a dropped one leaves a stale entry behind. + +`THIRD_PARTY.md` is informational and is not the licensing compliance artifact. The source +release, the sdist and the PyPI wheel resolve these packages with pip rather than bundling them, +so per the ASF [licensing howto](https://infra.apache.org/licensing-howto.html) they need no +dependency entries in `LICENSE` or `NOTICE`. The Docker image does bundle them and therefore needs +an aggregated `LICENSE`; `THIRD_PARTY.md` is the input for assembling it. + ### Avoiding secrets in commits Install [git-secrets](https://github.com/awslabs/git-secrets) to prevent diff --git a/Makefile b/Makefile index 9aab2fd7..f99f71e6 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ install: build rm -rf dist clean: - rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist *.egg-info logs junit-py*.xml *.whl NOTICE.txt + rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist *.egg-info logs junit-py*.xml *.whl NOTICE.txt THIRD_PARTY.table THIRD_PARTY.next # Avoid conflicts between .pyc/pycache related files created by local Python interpreters and other interpreters in Docker python-caches-clean: @@ -74,10 +74,21 @@ coverage: develop uv run coverage run -m pytest tests/ uv run coverage html -# Regenerates THIRD_PARTY.md from the runtime dependencies. --isolated runs in a fresh environment +# Regenerates the dependency table in THIRD_PARTY.md from the runtime dependencies, leaving the +# hand-maintained sections above the marker untouched. --isolated runs in a fresh environment # without the develop/test extras, so tools like pytest and ruff stay out of the list. +# --with-system keeps setuptools and wheel, which pip-licenses otherwise drops even though wheel +# is a declared runtime dependency; pip-licenses and its own dependencies are ignored instead. licenses: check-uv - uv run --isolated --with pip-licenses pip-licenses --with-authors --with-urls --format=markdown --ignore-packages solr-orbit --output-file THIRD_PARTY.md + @grep -q GENERATED-TABLE-BELOW THIRD_PARTY.md || { echo "THIRD_PARTY.md has no GENERATED-TABLE-BELOW marker; refusing to overwrite it" >&2; exit 1; } + uv run --isolated --with pip-licenses pip-licenses --with-authors --with-urls --with-system \ + --format=markdown --ignore-packages solr-orbit pip-licenses prettytable wcwidth \ + --output-file THIRD_PARTY.table + @awk '{ print } index($$0, "GENERATED-TABLE-BELOW") { exit }' THIRD_PARTY.md > THIRD_PARTY.next + @echo >> THIRD_PARTY.next + @cat THIRD_PARTY.table >> THIRD_PARTY.next + @mv THIRD_PARTY.next THIRD_PARTY.md + @rm -f THIRD_PARTY.table release-checks: ./release-checks.sh $(release_version) $(next_version) diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index 3f2d9b74..82535f10 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -1,3 +1,100 @@ +# Third-party dependencies + +## What this file is + +An informational inventory of Apache Solr Orbit's **runtime** dependencies, generated from +`uv.lock` with [pip-licenses](https://github.com/raimon49/pip-licenses) by `make licenses`. + +It is **not** the licensing compliance artifact. Per the ASF +[licensing howto](https://infra.apache.org/licensing-howto.html), `LICENSE` and `NOTICE` must +represent exactly what a distribution contains and nothing more: + +- The **source release** resolves these packages with pip at install time rather than bundling + them, so it needs no entries in `LICENSE` or `NOTICE` for them. +- The **Docker image** does bundle them (`uv pip install` into the image), so that artifact needs + an aggregated `LICENSE` and the propagated `NOTICE` content listed below. This file is the input + for assembling it, not a substitute. + +`NOTICE` records the Solr Orbit copyright and the source-derived attribution (OpenSearch, Rally); +dependency entries do not belong there unless the dependency itself ships a `NOTICE` whose content +must be propagated, and only for artifacts that bundle it. + +## Corrections to the generated table + +`pip-licenses` reports whatever a package declares, and many packages declare only a coarse trove +classifier such as `BSD License`. The values below were read from the license text each wheel +actually ships in its `dist-info`, and take precedence over the generated table: + +| Package | Generated value | Actual license | +|-----------------------------------------------------------------------------|---------------------------|----------------| +| google-crc32c | `UNKNOWN` | Apache-2.0 — the wheel carries no license metadata at all, but ships the full ALv2 text | +| Jinja2, cloudpickle, pandas, partd, pysolr, zict | `BSD License` / `BSD` | BSD-3-Clause | +| locket, pyasn1_modules | `BSD License` / `BSD` | BSD-2-Clause | +| google-auth, google-resumable-media, requests, s3transfer, sortedcontainers, tornado | `Apache Software License` | Apache-2.0 | +| python-dateutil | `Apache Software License; BSD License` | Dual Apache-2.0 **and** BSD-3-Clause; both texts are in the one license file and both apply | +| setuptools, wheel | *(absent before)* | MIT — `pip-licenses` filters these as "system packages" unless `--with-system` is passed, although `wheel` is a declared runtime dependency | + +## Dual licenses: the elections made + +Two dependencies are offered under a choice of licenses. Solr Orbit elects Apache-2.0 for both, +being an ASF project. Change this only by PMC decision, and update the Docker image's aggregated +`LICENSE` to match. + +| Package | Offered as | Elected | +|--------------|------------------------------|------------| +| cryptography | `Apache-2.0 OR BSD-3-Clause` | Apache-2.0 | +| packaging | `Apache-2.0 OR BSD-2-Clause` | Apache-2.0 | + +## Category B dependencies + +Two dependencies are [Category B](https://www.apache.org/legal/resolved.html#category-b), which +may be included in binary form only, and require a pointer to where the covered source can be +obtained: + +| Package | License | Note | +|---------|-------------------|------| +| certifi | MPL-2.0 | Source: https://github.com/certifi/python-certifi | +| tqdm | `MPL-2.0 AND MIT` | MPL-2.0 covers all files by default; MIT applies to named exceptions only. Source: https://github.com/tqdm/tqdm | + +No dependency is [Category X](https://www.apache.org/legal/resolved.html#category-x). + +## Dependencies that ship their own NOTICE + +Under ALv2 §4(d) the relevant content of a bundled dependency's `NOTICE` must be carried into the +notices of any artifact that bundles it. These four are the complete set among the runtime +dependencies: + +| Package | NOTICE content | +|------------|----------------| +| boto3 | A single copyright line | +| requests | A single copyright line | +| s3transfer | A single copyright line | +| botocore | Substantial: copyright plus the licenses of vendored requests and urllib3 and an MPL-2.0 CA-certificate bundle | + +## Dependencies that bundle further third-party code + +One license per package understates these; each ships several license files that an aggregated +`LICENSE` has to reproduce in full: + +| Package | Bundled licenses | +|-------------------|------------------| +| numpy | 17 license files (libdivide, highway, dragon4, x86-simd-sort, SVML, pocketfft, lapack_lite, random/*, pythoncapi-compat) | +| pandas | A 1000-line `LICENSE` covering roughly ten components (bottleneck, dateutil, musl, Sun/fdlibm, Arm, packaging, khash) | +| h5py | 7 license files (hdf5, pytables, python, stdint, lzf) | +| dask | Vendors numpy, with its license | +| pillow | Composite: MIT-CMU alongside other notices | +| typing_extensions | The CPython/PSF license stack | + +`pip-licenses --with-license-file --no-license-path` dumps these texts, and is the right starting +point for the Docker image's aggregated `LICENSE`. + +## Dependency table + +Generated by `make licenses` and replaced wholesale on every run — edit the sections above, not +the table. + + + | Name | Version | License | Author | URL | |---------------------------|-------------|----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------| | Faker | 40.39.0 | MIT License | joke2k | https://github.com/joke2k/faker | @@ -49,6 +146,7 @@ | requests | 2.34.2 | Apache Software License | Kenneth Reitz | https://github.com/psf/requests | | rpds-py | 2026.5.1 | MIT | Julian Berman | https://github.com/crate-py/rpds | | s3transfer | 0.19.0 | Apache Software License | Amazon Web Services | https://github.com/boto/s3transfer | +| setuptools | 82.0.1 | MIT | Python Packaging Authority | https://github.com/pypa/setuptools | | six | 1.17.0 | MIT License | Benjamin Peterson | https://github.com/benjaminp/six | | sortedcontainers | 2.4.0 | Apache Software License | Grant Jenks | http://www.grantjenks.com/docs/sortedcontainers/ | | tabulate | 0.10.0 | MIT | Sergey Astanin | https://github.com/astanin/python-tabulate | @@ -60,6 +158,7 @@ | typing-inspection | 0.4.4 | MIT | Victorien Plot | https://github.com/pydantic/typing-inspection | | typing_extensions | 4.16.0 | PSF-2.0 | "Guido van Rossum, Jukka Lehtosalo, Łukasz Langa, Michael Lee" | https://github.com/python/typing_extensions | | urllib3 | 2.8.0 | MIT | Andrey Petrov | https://github.com/urllib3/urllib3/blob/main/CHANGES.rst | +| wheel | 0.47.0 | MIT | Daniel Holth | https://github.com/pypa/wheel | | xyzservices | 2026.3.0 | BSD-3-Clause | Dani Arribas-Bel , Martin Fleischmann | https://github.com/geopandas/xyzservices | | yappi | 1.7.6 | MIT License | Sümer Cip | https://github.com/sumerc/yappi | | zict | 3.0.0 | BSD License | UNKNOWN | http://zict.readthedocs.io/en/latest/ | diff --git a/pyproject.toml b/pyproject.toml index dc7062a3..69b3bac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,77 +45,42 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] dependencies = [ - # License: BSD + # Licenses for these and their transitive dependencies are inventoried in THIRD_PARTY.md, + # regenerated with `make licenses`. Do not duplicate license information here. # Solr HTTP client for data operations (indexing, search, commit, optimize) "pysolr>=3.10.0", - # License: Apache 2.0 # HTTP client for Solr V2 API admin operations - # transitive dependencies: - # urllib3: MIT "requests>=2.28.0", - # License: BSD "psutil>=5.8.0", - # License: MIT "py-cpuinfo>=7.0.0", - # License: MIT "tabulate>=0.9.0", - # License: MIT "jsonschema>=3.1.1", - # License: BSD "Jinja2>=3.1.3", - # License: BSD "markupsafe>=2.0.1", - # License: MIT # With 3.10.7, we get InvalidActorAddress exception while initialize Actor "thespian>=3.10.1,<4.0.2", # always use the latest version, these are certificate files... - # License: MPL 2.0 "certifi", - # License: Apache 2.0 "yappi>=1.4.0", - # License: BSD "ijson>=2.6.1", - # License: Apache 2.0 - # transitive dependencies: - # google-crc32c: Apache 2.0 "google-resumable-media>=1.1.0", - # License: Apache 2.0 "google-auth>=1.22.1", - # License: MIT "wheel>=0.38.4", - # License: Apache 2.0 - # transitive dependencies: - # botocore: Apache 2.0 - # jmespath: MIT - # s3transfer: Apache 2.0 "boto3>=1.28.62", - # Licence: BSD-3-Clause "zstandard>=0.22.0", - # License: BSD # Required for knnvector workload "h5py>=3.10.0", - # License: BSD # Required for knnvector workload "numpy>=1.24.2,<=2.5.3", - # License: MIT "tqdm", - # License: MIT "faker", - # License: BSD "pandas>=1.4.3", - # License: MIT "mimesis==22.2.0", - # Licence: BSD-3-Clause "dask", - # Licence: BSD-3-Clause "dask[distributed]", - # Licence: BSD-3-Clause "bokeh!=3.0.*,>=2.4.2", - # License: MIT "pydantic>=2.10.6", - # License: MIT "pydantic_core>=2.27.2", - # License: MIT "PyYAML>=5.4", ]