From 7af0f79f367447cf117d6efa84973a954ce113b1 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 10:06:29 -0500 Subject: [PATCH 01/14] Build docs with sphinx-build -j auto Parallel reading and writing gives identical HTML and LaTeX output and saves a few seconds per build. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c284a57f788..cf29a7b677e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -198,7 +198,7 @@ jobs: name: stubs path: circuitpython-stubs/dist/* - name: Test Documentation Build (HTML) - run: sphinx-build -E -W -b html -D version="$CP_VERSION" -D release="$CP_VERSION" . _build/html + run: sphinx-build -E -W -j auto -b html -D version="$CP_VERSION" -D release="$CP_VERSION" . _build/html - uses: actions/upload-artifact@v7 with: name: docs-html diff --git a/Makefile b/Makefile index e7150281352..a71624a41ba 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ # You can set these variables from the command line. PYTHON = python3 -SPHINXOPTS = -W --keep-going +SPHINXOPTS = -W --keep-going -j auto SPHINXBUILD = sphinx-build PAPER = # path to build the generated docs From d930dcb516eafb79bfcf760be0ffadc31a4cce99 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 10:29:39 -0500 Subject: [PATCH 02/14] Move shared_bindings_matrix.py from docs/ to tools/ Six scripts in tools/ import it, each by adding docs/ to sys.path. In tools/, they can import it directly. conf.py adds tools/ to sys.path. ci_set_matrix.py: keep treating the file as docs-only: it triggers the docs build, but not board builds or the Zephyr tests. Co-Authored-By: Claude Opus 5.5 --- conf.py | 1 + .../build_board_specific_stubs/board_stub_builder.py | 2 +- tools/board_summaries.py | 3 --- tools/board_to_port.py | 3 --- tools/build_board_info.py | 1 - tools/build_release_files.py | 1 - tools/ci_set_matrix.py | 6 +++--- {docs => tools}/shared_bindings_matrix.py | 0 8 files changed, 5 insertions(+), 12 deletions(-) rename {docs => tools}/shared_bindings_matrix.py (100%) diff --git a/conf.py b/conf.py index 379ac442f19..b5c8e6b3716 100644 --- a/conf.py +++ b/conf.py @@ -36,6 +36,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("docs")) +sys.path.insert(0, os.path.abspath("tools")) sys.path.insert(0, os.path.abspath(".")) import shared_bindings_matrix diff --git a/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py b/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py index fb6219f373b..be3cb099255 100644 --- a/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py +++ b/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py @@ -266,7 +266,7 @@ def build_stubs(circuitpython_dir, circuitpython_org_dir, export_dir, version="8 circuitpython_dir = circuitpython_dir + "/" sys.path.append(circuitpython_dir) - from docs import shared_bindings_matrix + from tools import shared_bindings_matrix if not os.path.exists(export_dir): os.makedirs(export_dir) diff --git a/tools/board_summaries.py b/tools/board_summaries.py index 3ea87e68d55..2085ec8057e 100644 --- a/tools/board_summaries.py +++ b/tools/board_summaries.py @@ -22,9 +22,6 @@ # import re -import sys - -sys.path.append("../docs") from shared_bindings_matrix import support_matrix_by_board diff --git a/tools/board_to_port.py b/tools/board_to_port.py index 325105f9873..44acdcc2842 100644 --- a/tools/board_to_port.py +++ b/tools/board_to_port.py @@ -5,10 +5,7 @@ # SPDX-License-Identifier: MIT import sys -from pathlib import Path -docs = Path(__file__).parent.parent / "docs" -sys.path.append(str(docs)) from shared_bindings_matrix import get_board_mapping board = sys.argv[1] diff --git a/tools/build_board_info.py b/tools/build_board_info.py index bd9452b2626..24531f516b8 100755 --- a/tools/build_board_info.py +++ b/tools/build_board_info.py @@ -19,7 +19,6 @@ sys.path.append("adabot") import adabot.github_requests as github -sys.path.append("../docs") from shared_bindings_matrix import ( SUPPORTED_PORTS, support_matrix_by_board, diff --git a/tools/build_release_files.py b/tools/build_release_files.py index 4394cf931d9..d2018d3b95c 100755 --- a/tools/build_release_files.py +++ b/tools/build_release_files.py @@ -16,7 +16,6 @@ import json import tomllib -sys.path.append("../docs") from shared_bindings_matrix import get_settings_from_makefile TOP = pathlib.Path(__file__).resolve().parent.parent diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 971583e28d5..8056a3966ad 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -35,7 +35,6 @@ top_dir = tools_dir.parent sys.path.insert(0, str(tools_dir / "adabot")) -sys.path.insert(0, str(top_dir / "docs")) import build_board_info from shared_bindings_matrix import ( @@ -52,6 +51,7 @@ "tools/ci_changes_per_commit.py", "tools/ci_check_duplicate_usb_vid_pid.py", "tools/ci_set_matrix.py", + "tools/shared_bindings_matrix.py", ".github/workflows/run-tests.yml", ".github/workflows/run-zephyr-tests.yml", ".github/workflows/build-board-custom.yml", @@ -67,7 +67,7 @@ PATTERN_DOCS = ( r"^(?:\.github|docs|extmod\/ulab)|" - r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|\.readthedocs\.yml|conf\.py|requirements-doc\.txt)$|" + r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|tools\/shared_bindings_matrix\.py|\.readthedocs\.yml|conf\.py|requirements-doc\.txt)$|" r"(?:-stubs|\.(?:md|MD|mk|rst|RST)|/Makefile)$" ) @@ -78,7 +78,7 @@ # (this port has none), documentation, the unix test suite or the autogen PR workflow. PATTERN_ZEPHYR_TESTS_IGNORE = re.compile( r"^(?:docs|frozen|locale|tests)/|^ports/(?!zephyr-cp/)" - r"|^\.github/workflows/zephyr-autogen-pr\.yml$" + r"|^\.github/workflows/zephyr-autogen-pr\.yml$|^tools/shared_bindings_matrix\.py$" ) # Files that can't affect the unix coverage tests: other ports and the documentation. diff --git a/docs/shared_bindings_matrix.py b/tools/shared_bindings_matrix.py similarity index 100% rename from docs/shared_bindings_matrix.py rename to tools/shared_bindings_matrix.py From 2172cff9517242a36bf179e1739881dc770e06d8 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 10:35:17 -0500 Subject: [PATCH 03/14] shared_bindings_matrix: collect board data separately from formatting it collect_board_data() gathers everything about each board once, as JSON types. support_matrix_by_board() keeps its signature but now only formats that data, collecting it first if no board_data is passed. zephyr-cp boards have no mpconfigboard.h or .mk, so they now get empty MCU and flash values instead of raising FileNotFoundError when add_chips is set. Co-Authored-By: Claude Opus 5.5 --- tools/shared_bindings_matrix.py | 308 +++++++++++++++++--------------- 1 file changed, 160 insertions(+), 148 deletions(-) diff --git a/tools/shared_bindings_matrix.py b/tools/shared_bindings_matrix.py index b80566283ac..e6343f24fa0 100644 --- a/tools/shared_bindings_matrix.py +++ b/tools/shared_bindings_matrix.py @@ -318,173 +318,185 @@ def lookup_setting(settings, key, default=""): return value -def support_matrix_by_board( - use_branded_name=True, - withurl=True, - add_port=False, - add_chips=False, - add_pins=False, - add_branded_name=False, -): - """Compiles a list of the available core modules available for each - board. +def get_branded_name(board_id, board_directory, autogen_board_info): + """Return the board's product name, falling back to its board id.""" + if autogen_board_info: + # Many zephyr-cp boards share a name with a native board, so tell them apart. + return autogen_board_info["name"] + " (Zephyr)" + with open(board_directory / "mpconfigboard.h") as f: + board_name_re = re.search(r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", f.read()) + if not board_name_re: + return board_id + branded_name = board_name_re.group(1).strip('"') + if '"' in branded_name: # sometimes the closing " is not at line end + branded_name = branded_name[: branded_name.index('"')] + return branded_name + + +def get_mcu_and_flash(board_directory, autogen_board_info): + """Return the board's MCU name and its external flash chips as a quoted, comma-separated string. + + zephyr-cp boards don't record either yet, so they get empty strings. + """ + if autogen_board_info: + return "", "" + with open(board_directory / "mpconfigboard.h") as f: + mcu_re = re.search(r"(?<=MICROPY_HW_MCU_NAME)\s+(.+)", f.read()) + mcu = "" + if mcu_re: + mcu = mcu_re.group(1).strip('"') + if '"' in mcu: # in case the closing " is not at line end + mcu = mcu[: mcu.index('"')] + with open(board_directory / "mpconfigboard.mk") as f: + flash_re = re.search(r"(?<=EXTERNAL_FLASH_DEVICES)\s+=\s+(.+)", f.read()) + flash = "" + if flash_re: + # Multiple flash chips are written in several ways. Normalize them to + # a quoted, comma-separated string. + flash = '"' + flash_re.group(1).replace('"', "") + '"' + return mcu, flash + + +def get_pins(board_directory): + """Return the board's [board_pin, chip_pin] pairs from pins.c.""" + try: + with open(board_directory / "pins.c") as f: + pin_lines = f.readlines() + except FileNotFoundError: # silabs and zephyr-cp boards have no pins.c + return [] + pins = [] + for line in pin_lines: + if pin_re := re.search(r"QSTR_([^\)]+).+pin_([^\)]+)", line): + pins.append([pin_re.group(1), pin_re.group(2)]) + return pins + + +def get_board_modules(settings, autogen_board_info, base): + """Return the sorted names of the modules enabled on the board.""" + if autogen_board_info: + modules = [k for k, enabled in autogen_board_info["modules"].items() if enabled] + else: + modules = [ + info["name"] + for info in base.values() + if int(lookup_setting(settings, info["key"], "0")) + ] + return sorted(modules) + + +def get_board_extensions(board_id, settings): + """Return the file extensions the board's build produces, such as ["uf2"].""" + if "CIRCUITPY_BUILD_EXTENSIONS" not in settings: + raise OSError(f"Board extensions undefined: {board_id}.") + extensions = settings["CIRCUITPY_BUILD_EXTENSIONS"] + if isinstance(extensions, str): + extensions = [extension.strip() for extension in extensions.split(",")] + return extensions + + +def collect_board_data(): + """Gather what support_matrix_by_board() needs about every board, keyed by board id. + + Aliases are left out; support_matrix_by_board() adds them. The result holds + only JSON types, so it can be saved and reused. + + This runs make once for each board outside zephyr-cp, which takes a while. """ base = build_module_map() - def support_matrix(arg): + def collect(arg): board_id, board_info = arg port = board_info["port"] board_directory = board_info["directory"] - port_dir = board_directory.parent.parent - if port != "zephyr-cp": - settings = get_settings_from_makefile(str(port_dir), board_directory.name) - autogen_board_info = None - else: - circuitpython_toml_fn = board_directory / "circuitpython.toml" - with circuitpython_toml_fn.open("rb") as f: + if port == "zephyr-cp": + with (board_directory / "circuitpython.toml").open("rb") as f: settings = tomllib.load(f) - - autogen_board_info_fn = board_directory / "autogen_board_info.toml" - with autogen_board_info_fn.open("rb") as f: + with (board_directory / "autogen_board_info.toml").open("rb") as f: autogen_board_info = tomllib.load(f) - - if use_branded_name or add_branded_name: - if autogen_board_info: - # Many zephyr-cp boards share a name with a native board, so tell them apart. - branded_name = autogen_board_info["name"] + " (Zephyr)" - else: - with open(board_directory / "mpconfigboard.h") as get_name: - board_contents = get_name.read() - board_name_re = re.search(r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_contents) - if board_name_re: - branded_name = board_name_re.group(1).strip('"') - if '"' in branded_name: # sometimes the closing " is not at line end - branded_name = branded_name[: branded_name.index('"')] - board_name = branded_name - - if use_branded_name: - board_name = branded_name else: - board_name = board_id + port_dir = board_directory.parent.parent + settings = get_settings_from_makefile(str(port_dir), board_directory.name) + autogen_board_info = None - if add_chips: - with open(board_directory / "mpconfigboard.h") as get_name: - board_contents = get_name.read() - mcu_re = re.search(r"(?<=MICROPY_HW_MCU_NAME)\s+(.+)", board_contents) - if mcu_re: - mcu = mcu_re.group(1).strip('"') - if '"' in mcu: # in case the closing " is not at line end - mcu = mcu[: mcu.index('"')] - else: - mcu = "" - with open(board_directory / "mpconfigboard.mk") as get_name: - board_contents = get_name.read() - flash_re = re.search(r"(?<=EXTERNAL_FLASH_DEVICES)\s+=\s+(.+)", board_contents) - if flash_re: - # deal with the variability in the way multiple flash chips - # are denoted. We want them to end up as a quoted, - # comma separated string - flash = flash_re.group(1).replace('"', "") - flash = f'"{flash}"' - else: - flash = "" + frozen_libraries = [] + if "FROZEN_MPY_DIRS" in settings: + frozen_libraries = [ + list(library) + for library in sorted( + frozen_modules_from_dirs(settings["FROZEN_MPY_DIRS"], withurl=True) + ) + ] + + mcu, flash = get_mcu_and_flash(board_directory, autogen_board_info) + return board_id, { + "port": port, + "branded_name": get_branded_name(board_id, board_directory, autogen_board_info), + "mcu": mcu, + "flash": flash, + "pins": get_pins(board_directory), + "modules": get_board_modules(settings, autogen_board_info, base), + "extensions": get_board_extensions(board_id, settings), + # [name, repository url] pairs + "frozen_libraries": frozen_libraries, + } - if add_pins: - pins = [] - try: - with open(board_directory / "pins.c") as get_name: - pin_lines = get_name.readlines() - except FileNotFoundError: # silabs boards have no pins.c - pass - else: - for p in pin_lines: - pin_re = re.search(r"QSTR_([^\)]+).+pin_([^\)]+)", p) - if pin_re: - board_pin = pin_re.group(1) - chip_pin = pin_re.group(2) - pins.append((board_pin, chip_pin)) - - board_modules = [] - if autogen_board_info: - autogen_modules = autogen_board_info["modules"] - for k in autogen_modules: - if autogen_modules[k]: - board_modules.append(k) - else: - for module in base: - key = base[module]["key"] - if int(lookup_setting(settings, key, "0")): - board_modules.append(base[module]["name"]) - board_modules.sort() - - if "CIRCUITPY_BUILD_EXTENSIONS" in settings: - board_extensions = settings["CIRCUITPY_BUILD_EXTENSIONS"] - if isinstance(board_extensions, str): - board_extensions = [extension.strip() for extension in board_extensions.split(",")] - else: - raise OSError(f"Board extensions undefined: {board_name}.") + real_boards = [ + (board_id, board_info) + for board_id, board_info in get_board_mapping().items() + if not board_info.get("alias", False) + ] + with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + return dict(executor.map(collect, real_boards)) - frozen_modules = [] - if "FROZEN_MPY_DIRS" in settings: - frozen_modules = frozen_modules_from_dirs(settings["FROZEN_MPY_DIRS"], withurl) - if frozen_modules: - frozen_modules.sort() - # generate alias boards too +def support_matrix_by_board( + use_branded_name=True, + withurl=True, + add_port=False, + add_chips=False, + add_pins=False, + add_branded_name=False, + board_data=None, +): + """Return each board's modules, frozen libraries and build extensions. - board_info = { - "modules": board_modules, - "frozen_libraries": frozen_modules, - "extensions": board_extensions, + Aliases get their own entries. Keys are board ids, or product names if + use_branded_name is set. The add_* flags add more fields. + + board_data is the output of collect_board_data(). If it is None, it is collected here. + """ + if board_data is None: + board_data = collect_board_data() + + def board_entry(data): + entry = { + "modules": data["modules"], + "frozen_libraries": [ + tuple(library) if withurl else library[0] for library in data["frozen_libraries"] + ], + "extensions": data["extensions"], } if add_branded_name: - board_info["branded_name"] = branded_name + entry["branded_name"] = data["branded_name"] if add_port: - board_info["port"] = port + entry["port"] = data["port"] if add_chips: - board_info["mcu"] = mcu - board_info["flash"] = flash + entry["mcu"] = data["mcu"] + entry["flash"] = data["flash"] if add_pins: - board_info["pins"] = pins - board_matrix = [(board_name, board_info)] - if board_id in ALIASES_BY_BOARD: - for alias in ALIASES_BY_BOARD[board_id]: - if use_branded_name: - if alias in ALIASES_BRAND_NAMES: - alias = ALIASES_BRAND_NAMES[alias] - else: - alias = alias.replace("_", " ").title() - board_info = { - "modules": board_modules, - "frozen_libraries": frozen_modules, - "extensions": board_extensions, - } - if add_branded_name: - board_info["branded_name"] = branded_name - if add_port: - board_info["port"] = port - if add_chips: - board_info["mcu"] = mcu - board_info["flash"] = flash - if add_pins: - board_info["pins"] = pins - board_matrix.append((alias, board_info)) - - return board_matrix # this is now a list of (board,modules) - - board_mapping = get_board_mapping() - real_boards = [] - for board in board_mapping: - if not board_mapping[board].get("alias", False): - real_boards.append((board, board_mapping[board])) - executor = ThreadPoolExecutor(max_workers=os.cpu_count()) - mapped_exec = executor.map(support_matrix, real_boards) - # flatmap with comprehensions - boards = dict( - sorted([board for matrix in mapped_exec for board in matrix], key=lambda x: x[0]) - ) - - return boards + entry["pins"] = [tuple(pin) for pin in data["pins"]] + return entry + + boards = [] + for board_id, data in board_data.items(): + boards.append((data["branded_name"] if use_branded_name else board_id, board_entry(data))) + for alias in ALIASES_BY_BOARD.get(board_id, []): + if use_branded_name: + alias = ALIASES_BRAND_NAMES.get(alias, alias.replace("_", " ").title()) + boards.append((alias, board_entry(data))) + + # Several boards can share a product name; the last one listed wins. + return dict(sorted(boards, key=lambda board: board[0])) if __name__ == "__main__": From 0794f3379fdb4f32d9c7caf8ac297338f65843be Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 10:37:35 -0500 Subject: [PATCH 04/14] shared_bindings_matrix: save and load collected board data as JSON Co-Authored-By: Claude Opus 5.5 --- tools/shared_bindings_matrix.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/shared_bindings_matrix.py b/tools/shared_bindings_matrix.py index e6343f24fa0..c92487a4ca9 100644 --- a/tools/shared_bindings_matrix.py +++ b/tools/shared_bindings_matrix.py @@ -449,6 +449,22 @@ def collect(arg): return dict(executor.map(collect, real_boards)) +def write_board_data(path, board_data): + """Save the output of collect_board_data() as JSON.""" + # Keep the board order: when boards share a product name, the last one wins. + with open(path, "w") as f: + json.dump(board_data, f, indent=1) + + +def read_board_data(path): + """Return board data saved by write_board_data(), or None if path doesn't exist.""" + try: + with open(path) as f: + return json.load(f) + except FileNotFoundError: + return None + + def support_matrix_by_board( use_branded_name=True, withurl=True, From 26c499fdeea1d95ac9a4fb9093c5193550bd3f18 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 11:47:59 -0500 Subject: [PATCH 05/14] Board stubs: remove the NVM size line The NVM size in board stub docstrings has been wrong for many boards: - SAMD51 boards showed 256, the SAMD21 value. The regex took the first define in atmel-samd's mpconfigport.h, ignoring #ifdefs. - A board-level CIRCUITPY_INTERNAL_NVM_SIZE in mpconfigboard.h was never used: the code searched the port's file a second time instead. The value is a C macro, so only the C preprocessor can compute it. Stub builds don't have the port toolchains for that. The line was only informational, so drop it. Co-Authored-By: Claude Opus 5.5 --- .../board_stub_builder.py | 21 ------------------- .../circuitpython_setboard/__init__.py | 1 - 2 files changed, 22 deletions(-) diff --git a/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py b/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py index be3cb099255..8ead3695fda 100644 --- a/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py +++ b/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py @@ -160,7 +160,6 @@ def create_board_stubs(board_id, records, mappings, board_filename): boards_file.write(f"Board stub for {mappings['board_name']}\n") boards_file.write(f" - port: {mappings['port']}\n") boards_file.write(f" - board_id: {board_id}\n") - boards_file.write(f" - NVM size: {mappings['nvm_size']}\n") boards_file.write(f" - Included modules: {included_modules}\n") boards_file.write(f" - Frozen libraries: {frozen_libraries}\n") boards_file.write('"""\n\n') @@ -306,15 +305,6 @@ def build_stubs(circuitpython_dir, circuitpython_org_dir, export_dir, version="8 board_mappings[board]["pins_filename"] = pins_path - nvm_size = "Unknown" - with open(f"{port_path}/mpconfigport.h") as get_name: - port_contents = get_name.read() - port_nvm_re = re.search( - r"(?<=#define CIRCUITPY_INTERNAL_NVM_SIZE)\s+(.+)", port_contents - ) - if port_nvm_re: - nvm_size = port_nvm_re.group(1).strip() - board_name = board with open(f"{board_path}/mpconfigboard.h") as get_name: board_contents = get_name.read() @@ -322,18 +312,7 @@ def build_stubs(circuitpython_dir, circuitpython_org_dir, export_dir, version="8 if board_name_re: board_name = board_name_re.group(1).strip('"') - port_nvm_re = re.search( - r"(?<=#define CIRCUITPY_INTERNAL_NVM_SIZE)\s+(.+)", port_contents - ) - if port_nvm_re: - nvm_size = port_nvm_re.group(1).strip() - - nvm_size_re = re.search(r"^[0-9\(\) *]*$", nvm_size) - if nvm_size_re: - nvm_size = eval(nvm_size_re.group(0)) - board_mappings[board]["board_name"] = board_name - board_mappings[board]["nvm_size"] = nvm_size board_mappings[board]["libraries"] = libraries.get(board, None) process(board_mappings, export_dir) diff --git a/tools/board_stubs/circuitpython_setboard/__init__.py b/tools/board_stubs/circuitpython_setboard/__init__.py index 69a405405b8..f8754201b52 100644 --- a/tools/board_stubs/circuitpython_setboard/__init__.py +++ b/tools/board_stubs/circuitpython_setboard/__init__.py @@ -74,7 +74,6 @@ def set_board(): # Board stub for ... # - port: ... # - board_id: ... - # - NVM size: ... # - Included modules: ... # - Frozen libraries: ... # From d2c71b6b52a7c4bbf7c9cf6b8f9f855148ac2b02 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 11:49:29 -0500 Subject: [PATCH 06/14] Board stubs: use and save the collected board data board_stub_builder.py collects the board data once, uses it for both the support matrix and the board names, and saves it as circuitpython-stubs/board_data.json for conf.py to reuse. The names now come from the same parsing the docs use, which cuts at the closing quote. lolin_c3_mini and lolin_c3_pico stubs no longer include the trailing "// from Wemos MP" comment in the board name. Co-Authored-By: Claude Opus 5.5 --- .../board_stub_builder.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py b/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py index 8ead3695fda..7c80f0e476d 100644 --- a/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py +++ b/tools/board_stubs/build_board_specific_stubs/board_stub_builder.py @@ -260,7 +260,9 @@ def process(board_mappings, export_dir): print(f" - {board}") -def build_stubs(circuitpython_dir, circuitpython_org_dir, export_dir, version="8.2.9"): +def build_stubs( + circuitpython_dir, circuitpython_org_dir, export_dir, version="8.2.9", board_data_path=None +): if circuitpython_dir[-1] != "/": circuitpython_dir = circuitpython_dir + "/" @@ -270,10 +272,15 @@ def build_stubs(circuitpython_dir, circuitpython_org_dir, export_dir, version="8 if not os.path.exists(export_dir): os.makedirs(export_dir) + collected = shared_bindings_matrix.collect_board_data() + if board_data_path: + # conf.py reads this so it doesn't have to collect the same data again. + shared_bindings_matrix.write_board_data(board_data_path, collected) + libraries = {} if circuitpython_org_dir is None: libraries = shared_bindings_matrix.support_matrix_by_board( - use_branded_name=False, withurl=False + use_branded_name=False, withurl=False, board_data=collected ) else: with open(f"{circuitpython_org_dir}/_data/files.json") as libraries_file: @@ -305,18 +312,16 @@ def build_stubs(circuitpython_dir, circuitpython_org_dir, export_dir, version="8 board_mappings[board]["pins_filename"] = pins_path - board_name = board - with open(f"{board_path}/mpconfigboard.h") as get_name: - board_contents = get_name.read() - board_name_re = re.search(r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_contents) - if board_name_re: - board_name = board_name_re.group(1).strip('"') - - board_mappings[board]["board_name"] = board_name + board_mappings[board]["board_name"] = collected[lookup]["branded_name"] board_mappings[board]["libraries"] = libraries.get(board, None) process(board_mappings, export_dir) if __name__ == "__main__": - build_stubs("./", None, "circuitpython-stubs/board_definitions/") + build_stubs( + "./", + None, + "circuitpython-stubs/board_definitions/", + board_data_path="circuitpython-stubs/board_data.json", + ) From 3466b9ce1906000c464d3986561073f460585f0e Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 12:23:53 -0500 Subject: [PATCH 07/14] conf.py: reuse the board data saved by make stubs Before, the docs build collected the board data twice: once inside make stubs and again here. Each collection runs make once per board (about 15 seconds). A fresh HTML build drops from about 54 to 42 seconds. If the saved file is missing, support_matrix_by_board() still collects the data itself. The old comment said the stubs must be built before the matrix. The matrix no longer reads the stubs; now the matrix needs the saved data. Co-Authored-By: Claude Opus 5.5 --- conf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conf.py b/conf.py index b5c8e6b3716..277d4f355ad 100644 --- a/conf.py +++ b/conf.py @@ -46,11 +46,14 @@ # Grab the JSON values to use while building the module support matrix # in 'shared-bindings/index.rst' -# The stubs must be built before we calculate the shared bindings matrix +# Building the stubs also saves the board data the support matrix needs, so the +# slow per-board collection runs only once. If that file is missing, +# support_matrix_by_board() collects the data itself. subprocess.check_output(["make", "stubs"]) +board_data = shared_bindings_matrix.read_board_data("circuitpython-stubs/board_data.json") # modules_support_matrix = shared_bindings_matrix.support_matrix_excluded_boards() -modules_support_matrix = shared_bindings_matrix.support_matrix_by_board() +modules_support_matrix = shared_bindings_matrix.support_matrix_by_board(board_data=board_data) modules_support_matrix_reverse = defaultdict(list) for board, matrix_info in modules_support_matrix.items(): for module in matrix_info["modules"]: From 7fdf26bd8417b6fbcb201f121eefbc415cba73a3 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 12:29:09 -0500 Subject: [PATCH 08/14] board_summaries.py: iterate over csvs.items() write_csvs() iterated over the dict itself, so unpacking each key into (key, values) raised ValueError before any CSV was written. Co-Authored-By: Claude Opus 5.5 --- tools/board_summaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/board_summaries.py b/tools/board_summaries.py index 2085ec8057e..741c26dfdc2 100644 --- a/tools/board_summaries.py +++ b/tools/board_summaries.py @@ -182,7 +182,7 @@ def write_csvs(rows=1000, present="1", absent="0"): "frozen": frozen_incidence_matrix_csvs(s, rows=rows, present=present, absent=absent), "pins": board_pins_matrix_csvs(s, rows=rows), } - for key, values in csvs: + for key, values in csvs.items(): for i in range(len(values)): filename = f"{key}_{i}.csv" print(f"writing {filename}") From a53c218c3e267679601c190a1dcc433d0fabaea1 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 12:29:54 -0500 Subject: [PATCH 09/14] circuitpython_setboard: keep hyphens in port names The port was read by splitting the " - port: ..." line on every "-", so atmel-samd became "atmel". `circuitpython_setboard -l` then showed the wrong heading, and filtering by "atmel-samd" found nothing. Split on the first ":" instead. Co-Authored-By: Claude Opus 5.5 --- tools/board_stubs/circuitpython_setboard/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/board_stubs/circuitpython_setboard/__init__.py b/tools/board_stubs/circuitpython_setboard/__init__.py index f8754201b52..e5fb2d7c973 100644 --- a/tools/board_stubs/circuitpython_setboard/__init__.py +++ b/tools/board_stubs/circuitpython_setboard/__init__.py @@ -79,7 +79,7 @@ def set_board(): # lines = get_doc_or_exit(board).split("\n") - port = lines[2].split("-")[1].split(":")[1].strip() + port = lines[2].split(":", 1)[1].strip() if looking_for not in board.name.lower() and looking_for not in port.lower(): continue From 0f5c9682a80e9bd372f2227e4c4428fb630fbe81 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 12:35:11 -0500 Subject: [PATCH 10/14] ci_set_matrix.py: skip tests and mpy-cross for shared_bindings_matrix.py Under docs/, shared_bindings_matrix.py matched ^docs/ in the tests and mpy-cross ignore patterns. Its move to tools/ lost that. Add it to both patterns so it is still treated as docs-only. Co-Authored-By: Claude Opus 5.5 --- tools/ci_set_matrix.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 8056a3966ad..2164d08a1a7 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -81,15 +81,18 @@ r"|^\.github/workflows/zephyr-autogen-pr\.yml$|^tools/shared_bindings_matrix\.py$" ) -# Files that can't affect the unix coverage tests: other ports and the documentation. -# The tests build nothing from ports/ except ports/unix. frozen/ is not on this list, -# because run-tests.py puts the frozen asyncio and Ticks libraries on the module search path. -PATTERN_TESTS_IGNORE = re.compile(r"^docs/|^ports/(?!unix/)") - -# Files that can't affect mpy-cross: ports, frozen libraries, the documentation and the tests. -# The board builds don't need the mpy-cross and mpy-cross-mac jobs. They use the mpy-cross -# that the scheduler job builds. -PATTERN_MPY_CROSS_IGNORE = re.compile(r"^(?:docs|frozen|ports|tests)/") +# Files that can't affect the unix coverage tests: other ports and the documentation, +# including the docs' support matrix code. The tests build nothing from ports/ except +# ports/unix. frozen/ is not on this list, because run-tests.py puts the frozen asyncio and +# Ticks libraries on the module search path. +PATTERN_TESTS_IGNORE = re.compile(r"^docs/|^ports/(?!unix/)|^tools/shared_bindings_matrix\.py$") + +# Files that can't affect mpy-cross: ports, frozen libraries, the documentation (including +# the support matrix code) and the tests. The board builds don't need the mpy-cross and +# mpy-cross-mac jobs. They use the mpy-cross that the scheduler job builds. +PATTERN_MPY_CROSS_IGNORE = re.compile( + r"^(?:docs|frozen|ports|tests)/|^tools/shared_bindings_matrix\.py$" +) # Zephyr boards don't use make, so their module tables can't be computed here. Each # board's build writes autogen_board_info.toml next to its circuitpython.toml and that From e733efc956a4d0248db7b08dad96f837a622a5b6 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 12:35:35 -0500 Subject: [PATCH 11/14] ci_set_matrix.py: tools/board_stubs/ only affects the docs Changes under tools/board_stubs/ matched no rule, so CI built every board and skipped the docs build. That code only builds the board stubs, which the docs build makes. Build the docs for it, and skip the board builds, the Zephyr tests, the unix tests and mpy-cross. Co-Authored-By: Claude Opus 5.5 --- tools/ci_set_matrix.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 2164d08a1a7..d5928bff694 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -48,6 +48,7 @@ "conf.py", "docs", "tests", + "tools/board_stubs/", "tools/ci_changes_per_commit.py", "tools/ci_check_duplicate_usb_vid_pid.py", "tools/ci_set_matrix.py", @@ -66,7 +67,7 @@ } PATTERN_DOCS = ( - r"^(?:\.github|docs|extmod\/ulab)|" + r"^(?:\.github|docs|extmod\/ulab|tools\/board_stubs\/)|" r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|tools\/shared_bindings_matrix\.py|\.readthedocs\.yml|conf\.py|requirements-doc\.txt)$|" r"(?:-stubs|\.(?:md|MD|mk|rst|RST)|/Makefile)$" ) @@ -75,23 +76,27 @@ # The Zephyr tests build native_sim and the two bsim boards out of the shared sources, so a # change confined to these cannot reach them: another port, a translation, a frozen library -# (this port has none), documentation, the unix test suite or the autogen PR workflow. +# (this port has none), documentation, the board stub tools, the unix test suite or the +# autogen PR workflow. PATTERN_ZEPHYR_TESTS_IGNORE = re.compile( - r"^(?:docs|frozen|locale|tests)/|^ports/(?!zephyr-cp/)" + r"^(?:docs|frozen|locale|tests|tools/board_stubs)/|^ports/(?!zephyr-cp/)" r"|^\.github/workflows/zephyr-autogen-pr\.yml$|^tools/shared_bindings_matrix\.py$" ) # Files that can't affect the unix coverage tests: other ports and the documentation, -# including the docs' support matrix code. The tests build nothing from ports/ except -# ports/unix. frozen/ is not on this list, because run-tests.py puts the frozen asyncio and -# Ticks libraries on the module search path. -PATTERN_TESTS_IGNORE = re.compile(r"^docs/|^ports/(?!unix/)|^tools/shared_bindings_matrix\.py$") +# including the docs' support matrix code and the board stub tools. The tests build nothing +# from ports/ except ports/unix. frozen/ is not on this list, because run-tests.py puts the +# frozen asyncio and Ticks libraries on the module search path. +PATTERN_TESTS_IGNORE = re.compile( + r"^(?:docs|tools/board_stubs)/|^ports/(?!unix/)|^tools/shared_bindings_matrix\.py$" +) # Files that can't affect mpy-cross: ports, frozen libraries, the documentation (including -# the support matrix code) and the tests. The board builds don't need the mpy-cross and -# mpy-cross-mac jobs. They use the mpy-cross that the scheduler job builds. +# the support matrix code and the board stub tools) and the tests. The board builds don't +# need the mpy-cross and mpy-cross-mac jobs. They use the mpy-cross that the scheduler job +# builds. PATTERN_MPY_CROSS_IGNORE = re.compile( - r"^(?:docs|frozen|ports|tests)/|^tools/shared_bindings_matrix\.py$" + r"^(?:docs|frozen|ports|tests|tools/board_stubs)/|^tools/shared_bindings_matrix\.py$" ) # Zephyr boards don't use make, so their module tables can't be computed here. Each From 1acdf0a01501b4918111f25fcd688f453f983a9b Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 12:57:09 -0500 Subject: [PATCH 12/14] docs/README.md: how to build the PDF; SPHINXOPTS replaces the defaults The PDF build needs LaTeX and rsvg-convert, which were listed only in the CI workflow. Document the Debian/Ubuntu packages and make latexpdf. Setting SPHINXOPTS replaces the default -W --keep-going -j auto, so the example now includes them. Co-Authored-By: Claude Opus 5.5 --- docs/README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index cddc45b03b3..33f4b3f1821 100644 --- a/docs/README.md +++ b/docs/README.md @@ -23,6 +23,18 @@ In `circuitpython/`, build the docs: You'll find the index page at `_build/html/index.html`. +### Building the PDF + +The PDF build also needs LaTeX and `rsvg-convert`. On Debian or Ubuntu, install them with: + + sudo apt install latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra + +Then build the PDF: + + make latexpdf + +You'll find it at `_build/latex/CircuitPython.pdf`. + ### More flexibility Running `make` by itself will list out the multiple doc generating commands available. @@ -38,8 +50,9 @@ All commands will, by default, run with `-E` (forces a rebuild from scratch of d # will turn OFF the force rebuild and make it doubly verbose when running make html FORCE= VERBOSE="-v -v" -You can also pass other options to sphinx by using `SPHINXOPTS`. +You can also pass other options to sphinx by using `SPHINXOPTS`. It replaces the default +options, `-W --keep-going -j auto`, so include any of those you still want: - make html SPHINXOPTS="-T" + make html SPHINXOPTS="-W --keep-going -j auto -T" For more flexibility and customization, take a look at the Makefile for all variables you can pass in and overwrite. From ce8f65948b2eb9e48408014c883385b9b68998b3 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 13:09:01 -0500 Subject: [PATCH 13/14] make latexpdf: run latexmk with -silent pdflatex's first pass prints a warning for every cross-reference, about 3,300 lines, because nothing is resolved until a later pass. -silent sends the pdflatex output to the .log file only. latexmk still reports references left undefined after the last pass, and still fails on LaTeX errors. Use "make latexpdf LATEXMKOPTS=" to see the full output. Co-Authored-By: Claude Opus 5.5 --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a71624a41ba..ca0539fd4ee 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ PYTHON = python3 SPHINXOPTS = -W --keep-going -j auto SPHINXBUILD = sphinx-build +# -silent hides pdflatex's progress output, including the thousands of undefined-reference +# warnings from its first pass. Warnings left after the last pass, and errors, are still reported. +LATEXMKOPTS = -silent PAPER = # path to build the generated docs BUILDDIR = _build @@ -171,7 +174,7 @@ latex: latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf + $(MAKE) -C $(BUILDDIR)/latex all-pdf LATEXMKOPTS="$(LATEXMKOPTS)" @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." # seems to be malfunctioning From c5bff674c381b0b664da7d6625900c7c50b66d79 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Thu, 19 Nov 2026 13:19:46 -0500 Subject: [PATCH 14/14] docs/README.md: where make latexpdf's pdflatex output goes Co-Authored-By: Claude Opus 5.5 --- docs/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/README.md b/docs/README.md index 33f4b3f1821..fb29feb7c21 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,6 +35,11 @@ Then build the PDF: You'll find it at `_build/latex/CircuitPython.pdf`. +`pdflatex`'s output is verbose, with thousands of inconsequential warnings, so it goes to +`_build/latex/CircuitPython.log` instead of the console. To see it on the console anyway: + + make latexpdf LATEXMKOPTS= + ### More flexibility Running `make` by itself will list out the multiple doc generating commands available.