diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e9b4be4..57ab64f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,9 +71,8 @@ function(_infini_ops_enable_legacy_options_from_plugins) elseif(_plugin STREQUAL "cuda-common") # Shared dependency plugin; no legacy device option to set. else() - message(FATAL_ERROR - "Unknown infini_ops plugin `${_plugin}`. v1 supports built-in plugins: " - "`cpu`, `nvidia`, `iluvatar`, `hygon`, `metax`, `moore`, `cambricon`, `ascend`, `cuda-common`") + # External plugins are validated when `infini_ops_enable_plugin` + # resolves their `plugin.cmake` entry. endif() endforeach() endfunction() diff --git a/cmake/infini_ops_plugins.cmake b/cmake/infini_ops_plugins.cmake index 15242e40..5d4d8f88 100644 --- a/cmake/infini_ops_plugins.cmake +++ b/cmake/infini_ops_plugins.cmake @@ -3,12 +3,63 @@ include_guard(GLOBAL) set(INFINI_OPS_PLUGINS "" CACHE STRING "Comma- or semicolon-separated `infini_ops` build-time plugins to enable.") set(INFINI_OPS_PLUGIN_ROOT "${PROJECT_SOURCE_DIR}/plugins" CACHE PATH - "Directory containing `infini_ops` build-time plugins.") + "Directory containing built-in `infini_ops` build-time plugins.") +set(INFINI_OPS_PLUGIN_ROOTS "" CACHE STRING + "Additional comma- or semicolon-separated `infini_ops` build-time plugin roots.") set(INFINI_OPS_PLUGIN_CONTRACT_VERSION 1) set(_INFINI_OPS_KNOWN_DEVICE_PLUGINS cpu nvidia iluvatar hygon metax moore cambricon ascend) +function(_infini_ops_get_plugin_roots out_var) + set(_roots "${INFINI_OPS_PLUGIN_ROOT}") + + if(INFINI_OPS_PLUGIN_ROOTS) + set(_extra_roots "${INFINI_OPS_PLUGIN_ROOTS}") + string(REPLACE "," ";" _extra_roots "${_extra_roots}") + foreach(_root IN LISTS _extra_roots) + string(STRIP "${_root}" _root) + if(NOT _root STREQUAL "") + list(APPEND _roots "${_root}") + endif() + endforeach() + endif() + + if(_roots) + list(REMOVE_DUPLICATES _roots) + endif() + + set(${out_var} ${_roots} PARENT_SCOPE) +endfunction() + +function(_infini_ops_find_plugin_entry name out_var) + _infini_ops_get_plugin_roots(_plugin_roots) + set(_matches) + + foreach(_root IN LISTS _plugin_roots) + set(_candidate "${_root}/${name}/plugin.cmake") + if(EXISTS "${_candidate}") + list(APPEND _matches "${_candidate}") + endif() + endforeach() + + list(LENGTH _matches _match_count) + if(_match_count EQUAL 0) + string(REPLACE ";" "`, `" _roots_message "${_plugin_roots}") + message(FATAL_ERROR + "`infini_ops` plugin `${name}` `CMake` entry was not found in roots: " + "`${_roots_message}`.") + elseif(_match_count GREATER 1) + string(REPLACE ";" "`, `" _matches_message "${_matches}") + message(FATAL_ERROR + "`infini_ops` plugin `${name}` has duplicate `CMake` entries: " + "`${_matches_message}`.") + endif() + + list(GET _matches 0 _entry_path) + set(${out_var} "${_entry_path}" PARENT_SCOPE) +endfunction() + function(_infini_ops_append_unique_global property_name) get_property(_values GLOBAL PROPERTY "${property_name}") foreach(_value ${ARGN}) @@ -96,10 +147,7 @@ function(infini_ops_enable_plugin name) message(FATAL_ERROR "`infini_ops` plugin dependency cycle detected: `${_cycle}`.") endif() - set(_entry_path "${INFINI_OPS_PLUGIN_ROOT}/${name}/plugin.cmake") - if(NOT EXISTS "${_entry_path}") - message(FATAL_ERROR "`infini_ops` plugin `${name}` `CMake` entry was not found: `${_entry_path}`.") - endif() + _infini_ops_find_plugin_entry("${name}" _entry_path) set_property(GLOBAL APPEND PROPERTY INFINI_OPS_PLUGIN_LOADING_STACK "${name}") include("${_entry_path}") diff --git a/docs/plugin_contract.md b/docs/plugin_contract.md index c70c7853..0ceb65d6 100644 --- a/docs/plugin_contract.md +++ b/docs/plugin_contract.md @@ -61,7 +61,7 @@ Platform entries should keep platform-specific compile definitions, source lists ## Selection -`INFINI_OPS_PLUGINS` is the canonical configure option for selecting plugins. Legacy options such as `WITH_NVIDIA`, `WITH_ASCEND`, and `WITH_CAMBRICON` remain compatibility shims and map onto the same plugin enable path. +`INFINI_OPS_PLUGINS` is the canonical configure option for selecting plugins. `INFINI_OPS_PLUGIN_ROOT` points at the built-in plugin root by default, and `INFINI_OPS_PLUGIN_ROOTS` can add comma- or semicolon-separated external plugin roots. Legacy options such as `WITH_NVIDIA`, `WITH_ASCEND`, and `WITH_CAMBRICON` remain compatibility shims and map onto the same plugin enable path. `cuda-common` is a `shared` plugin. It is enabled through dependencies from CUDA-like device plugins such as `nvidia`, `iluvatar`, `metax`, `moore`, and `hygon`; it does not expose a standalone user device. diff --git a/scripts/infini_ops_plugin_registry.py b/scripts/infini_ops_plugin_registry.py index dbb15e20..b2968c3f 100644 --- a/scripts/infini_ops_plugin_registry.py +++ b/scripts/infini_ops_plugin_registry.py @@ -205,17 +205,41 @@ def _append_unique(values, new_values): values.append(value) -def load_plugin_manifests(plugin_root): - plugin_root = pathlib.Path(plugin_root) +def _iter_plugin_roots(plugin_roots): + if isinstance(plugin_roots, str | pathlib.PurePath): + raw_roots = [plugin_roots] + else: + raw_roots = plugin_roots - return { - path.parent.name: _load_manifest(path) - for path in sorted(plugin_root.glob("*/plugin.json")) - } + for root in raw_roots: + if isinstance(root, str): + parts = root.replace(",", ";").split(";") + else: + parts = [root] + + for part in parts: + if isinstance(part, str): + part = part.strip() + if part: + yield pathlib.Path(part) + + +def load_plugin_manifests(plugin_roots): + manifests = {} + + for plugin_root in _iter_plugin_roots(plugin_roots): + for path in sorted(plugin_root.glob("*/plugin.json")): + manifest = _load_manifest(path) + name = manifest["name"] + if name in manifests: + raise ValueError(f"Duplicate plugin `{name}` across plugin roots.") + manifests[name] = manifest + + return manifests -def load_plugin_registry(plugin_root, requested_plugins): - manifests = load_plugin_manifests(plugin_root) +def load_plugin_registry(plugin_roots, requested_plugins): + manifests = load_plugin_manifests(plugin_roots) ordered = [] visiting = [] diff --git a/scripts/infini_ops_plugin_test_matrix.py b/scripts/infini_ops_plugin_test_matrix.py index 4a5fd5aa..86d4120f 100644 --- a/scripts/infini_ops_plugin_test_matrix.py +++ b/scripts/infini_ops_plugin_test_matrix.py @@ -144,13 +144,17 @@ def main(argv=None): ) parser.add_argument( "--plugin-root", - default="plugins", - help="Directory containing built-in `plugin.json` manifests.", + action="append", + default=None, + help=( + "Directory containing `plugin.json` manifests. Pass multiple times " + "to include external plugin roots." + ), ) parser.add_argument("paths", nargs="*", help="Changed paths to classify.") args = parser.parse_args(argv) - matrix = build_test_matrix(args.plugin_root, _read_paths(args)) + matrix = build_test_matrix(args.plugin_root or ["plugins"], _read_paths(args)) print(json.dumps(matrix, indent=2, sort_keys=True)) diff --git a/tests/test_plugin_registry.py b/tests/test_plugin_registry.py index f0827b43..4edbdade 100644 --- a/tests/test_plugin_registry.py +++ b/tests/test_plugin_registry.py @@ -260,6 +260,42 @@ def test_load_plugins_rejects_dependency_cycles(tmp_path): raise AssertionError("cyclic plugin dependencies should be rejected") +def test_load_plugin_manifests_accepts_multiple_roots(tmp_path): + module = _load_registry_module() + builtin_root = tmp_path / "builtin" + external_root = tmp_path / "external" + builtin_root.mkdir() + external_root.mkdir() + _write_manifest(builtin_root, "cpu", _cpu_manifest()) + _write_manifest( + external_root, + "external-cpu", + _cpu_manifest(name="external-cpu"), + ) + + manifests = module.load_plugin_manifests([builtin_root, external_root]) + + assert list(manifests) == ["cpu", "external-cpu"] + + +def test_load_plugin_manifests_rejects_duplicate_plugin_names(tmp_path): + module = _load_registry_module() + first_root = tmp_path / "first" + second_root = tmp_path / "second" + first_root.mkdir() + second_root.mkdir() + _write_manifest(first_root, "cpu", _cpu_manifest()) + _write_manifest(second_root, "cpu", _cpu_manifest()) + + try: + module.load_plugin_manifests([first_root, second_root]) + except ValueError as exc: + assert "duplicate plugin" in str(exc).lower() + assert "`cpu`" in str(exc) + else: + raise AssertionError("duplicate plugin names across roots should be rejected") + + def test_builtin_plugin_manifests_load_individually(): module = _load_registry_module() plugin_root = pathlib.Path(__file__).resolve().parents[1] / "plugins" diff --git a/tests/test_plugin_test_matrix.py b/tests/test_plugin_test_matrix.py index d0ce683e..461ec296 100644 --- a/tests/test_plugin_test_matrix.py +++ b/tests/test_plugin_test_matrix.py @@ -99,6 +99,39 @@ def test_plugin_manifest_change_maps_to_that_plugin(): assert matrix["ci_platforms"] == ["cambricon"] +def test_matrix_accepts_multiple_plugin_roots(tmp_path): + module = _load_matrix_module() + external_root = tmp_path / "external" + plugin_dir = external_root / "external-cpu" + plugin_dir.mkdir(parents=True) + (plugin_dir / "plugin.cmake").write_text("# external plugin\n", encoding="utf-8") + (plugin_dir / "plugin.json").write_text( + json.dumps( + { + "name": "external-cpu", + "kind": "device", + "contract_version": 1, + "devices": ["cpu"], + "depends": [], + "cmake_entry": "plugin.cmake", + "source_roots": ["external/native/cpu"], + "operator_roots": ["external/native/cpu/ops"], + "device_headers": {"cpu": "external/native/cpu/device_.h"}, + "test_devices": {"cpu": "cpu"}, + } + ), + encoding="utf-8", + ) + + matrix = module.build_test_matrix( + [_plugin_root(), external_root], ["external/native/cpu/ops/add.cc"] + ) + + assert matrix["plugins"] == ["external-cpu"] + assert matrix["devices"] == ["cpu"] + assert matrix["ci_platforms"] == [] + + def test_cli_outputs_json_matrix(tmp_path): repo = pathlib.Path(__file__).resolve().parents[1] script = repo / "scripts" / "infini_ops_plugin_test_matrix.py"