diff --git a/packages/overture-schema-common/changelog.d/622.misc.md b/packages/overture-schema-common/changelog.d/622.misc.md new file mode 100644 index 000000000..e6e2aace4 --- /dev/null +++ b/packages/overture-schema-common/changelog.d/622.misc.md @@ -0,0 +1 @@ +Added `deepdiff` to the development dependency group. diff --git a/packages/overture-schema-common/pyproject.toml b/packages/overture-schema-common/pyproject.toml index e63298425..d652ccb74 100644 --- a/packages/overture-schema-common/pyproject.toml +++ b/packages/overture-schema-common/pyproject.toml @@ -30,6 +30,7 @@ packages = ["src/overture"] [dependency-groups] dev = [ + "deepdiff>=8.6.0", "jsonpath-ng>=1.7.0", "types-pyyaml>=6.0.12.20250516", "types-shapely>=2.1.0.20250710", diff --git a/packages/overture-schema-validation/README.md b/packages/overture-schema-validation/README.md new file mode 100644 index 000000000..41cda9f8b --- /dev/null +++ b/packages/overture-schema-validation/README.md @@ -0,0 +1,25 @@ +# overture-schema-validation + +Validate data against the union of all discovered Overture Maps models. + +This package provides `validate` and `validate_json`, which check a Python object or JSON document against every Overture model registered on the `overture.models` entry point and return the matching validated model instance. + +## Installation + +```bash +pip install overture-schema-validation +``` + +## Usage + +```python +from overture.schema.validation import validate, validate_json + +# Validate a Python object (a dict or a model instance) +feature = validate({"type": "segment", "id": "...", "geometry": "..."}) + +# Validate a JSON document +feature = validate_json('{"type": "segment", "id": "...", "geometry": "..."}') +``` + +Both raise `pydantic.ValidationError` when the input matches no model. Which models participate is resolved at runtime by entry-point discovery, so installing additional Overture theme packages widens what these functions accept. diff --git a/packages/overture-schema-validation/changelog.d/622.misc.md b/packages/overture-schema-validation/changelog.d/622.misc.md new file mode 100644 index 000000000..53379037c --- /dev/null +++ b/packages/overture-schema-validation/changelog.d/622.misc.md @@ -0,0 +1 @@ +Extracted `validate()` and `validate_json()` into the new `overture-schema-validation` package, off the shared `overture.schema` namespace root. diff --git a/packages/overture-schema-validation/changelog.d/README.md b/packages/overture-schema-validation/changelog.d/README.md new file mode 100644 index 000000000..83ffd44b1 --- /dev/null +++ b/packages/overture-schema-validation/changelog.d/README.md @@ -0,0 +1,17 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per change to this package (including patch-level fixes and internal +work): + +```text +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-validation/pyproject.toml b/packages/overture-schema-validation/pyproject.toml new file mode 100644 index 000000000..abc30a0c5 --- /dev/null +++ b/packages/overture-schema-validation/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +maintainers = [ + {name = "Overture Maps Schema Working Group"}, +] +name = "overture-schema-validation" +version = "0.1.1" +description = "Validation helpers for the union of all discovered Overture models" +requires-python = ">=3.10" +license = "MIT" +readme = "README.md" +dependencies = [ + "overture-schema-common", + "overture-schema-system", + "pydantic>=2.12.0", +] + +[project.urls] +Homepage = "https://overturemaps.org" +Source = "https://github.com/OvertureMaps/schema" +Issues = "https://github.com/OvertureMaps/schema/issues" + +[tool.uv.sources] +overture-schema-common = { workspace = true } +overture-schema-system = { workspace = true } + +[dependency-groups] +dev = [ + "pyyaml>=6.0.2", + "yamlcore>=0.0.4", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/overture"] diff --git a/packages/overture-schema-validation/src/overture/__init__.py b/packages/overture-schema-validation/src/overture/__init__.py new file mode 100644 index 000000000..8db66d3d0 --- /dev/null +++ b/packages/overture-schema-validation/src/overture/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/packages/overture-schema-validation/src/overture/schema/__init__.py b/packages/overture-schema-validation/src/overture/schema/__init__.py new file mode 100644 index 000000000..8db66d3d0 --- /dev/null +++ b/packages/overture-schema-validation/src/overture/schema/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/packages/overture-schema-validation/src/overture/schema/validation/__init__.py b/packages/overture-schema-validation/src/overture/schema/validation/__init__.py new file mode 100644 index 000000000..0cea00b0c --- /dev/null +++ b/packages/overture-schema-validation/src/overture/schema/validation/__init__.py @@ -0,0 +1,169 @@ +from collections.abc import Generator +from functools import reduce +from operator import or_ +from types import UnionType +from typing import Annotated, Any, Literal, cast, get_args, get_origin + +from pydantic import BaseModel, Field, Tag, TypeAdapter + +from overture.schema.common import OvertureFeature +from overture.schema.system.discovery import discover_models +from overture.schema.system.feature import Feature + + +def validate(data: object) -> BaseModel: + """ + Validate a Python object, which can be a dictionary or model instance, using the union of all + discovered Overture models. + + Parameters + ---------- + data : object + Python object to validate against the model. + + Returns + ------- + BaseModel + Validated model class + + Raises + ------ + ValidationError + If `data` is not valid according to one of the discovered Overture models + """ + tap = _union_type_adapter() + + return cast(BaseModel, tap.validate_python(data)) + + +def validate_json(json_data: str | bytes | bytearray) -> BaseModel: + """ + Validate JSON data using the union of all discovered Overture models. + + Parameters + ---------- + json_data : str | bytes | bytearray + JSON data to validate + + Returns + ------- + BaseModel + Validated model class + + Raises + ------ + ValidationError + If `json_data` is not valid according to one of the discovered Overture models + """ + tap = _union_type_adapter() + + return cast(BaseModel, tap.validate_json(json_data)) + + +__all__ = [ + "validate", + "validate_json", +] + + +def _union_type_adapter() -> TypeAdapter: + """ + Return a Pydantic type adapter that can validate the union of all models discovered using entry + points. + """ + models = discover_models() + if not models: + raise RuntimeError("no registered models found via entry points") + + discriminated_models: tuple[type[OvertureFeature], ...] = tuple( + cast(type[OvertureFeature], m) for m in models.values() if _can_discriminate(m) + ) + discriminated_union: UnionType | None = _discriminated_union(discriminated_models) + + non_discriminated_models: Generator[type[BaseModel], None, None] = ( + m for m in models.values() if not _can_discriminate(m) + ) + non_discriminated_union: UnionType | None = reduce( + or_, non_discriminated_models, None + ) + + if discriminated_union and non_discriminated_union: + model_union = discriminated_union | non_discriminated_union + elif discriminated_union: + model_union = discriminated_union + elif non_discriminated_union: + model_union = non_discriminated_union + else: + raise RuntimeError("logic error: unreachable code") + + return TypeAdapter(model_union) + + +def _discriminated_union( + feature_classes: tuple[type[OvertureFeature], ...], +) -> Any: # noqa: ANN401 + """ + Create a discriminated union of the Overture features since they can be discriminated on the + `type` field. This is just a performance optimization, and the union will work even if no models + are discriminated. + """ + if not feature_classes: + return None + else: + return Annotated[ + reduce( + or_, + ( + Annotated[f, Tag(cast(str, _typeliteral(f)))] + for f in feature_classes + ), + ), + Field(discriminator=Feature.field_discriminator("type", *feature_classes)), + ] + + +def _can_discriminate(model_class: object) -> bool: + """ + Return true if given value can participate in a discriminated union on the `type` field because + it is an Overture feature with where the `type` field has a single literal value. + """ + return ( + isinstance(model_class, type) + and issubclass(model_class, OvertureFeature) + and _typeliteral(cast(type[OvertureFeature], model_class)) is not None + ) + + +def _typeliteral(feature_class: type[OvertureFeature]) -> object: + """ + Return the literal value of the Overture Feature model's `type` field, if it has one, or `None` + if it does not. + + Parameters + ---------- + feature_class : type[OvertureFeature] + Overture feature model class + + Returns + ------- + object + The literal constrained value of the model class' `type` field, or `None` if the `type` + field does not have a literal value + + Raises + ------ + TypeError + If the `type` field is constrained to `Literal[None]`, as this is absurd + """ + type_type = feature_class.model_fields["type"].annotation + while get_origin(type_type) is Annotated: + type_type = get_args(type_type)[0] + if get_origin(type_type) is not Literal: + return None + literal = get_args(type_type)[0] + if literal is None: + raise TypeError( + f"literal value of `type` field for `{OvertureFeature.__name__}` class " + f"`{feature_class.__name__}` is constrained to `None`" + ) + return literal diff --git a/packages/overture-schema-validation/src/overture/schema/validation/py.typed b/packages/overture-schema-validation/src/overture/schema/validation/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema/tests/test_schema_validation.py b/packages/overture-schema-validation/tests/test_schema_validation.py similarity index 99% rename from packages/overture-schema/tests/test_schema_validation.py rename to packages/overture-schema-validation/tests/test_schema_validation.py index b3ec4399f..13702a5a9 100644 --- a/packages/overture-schema/tests/test_schema_validation.py +++ b/packages/overture-schema-validation/tests/test_schema_validation.py @@ -5,7 +5,7 @@ import pytest import yaml -from overture.schema import validate, validate_json +from overture.schema.validation import validate, validate_json from pydantic import ValidationError from yamlcore import CoreLoader # type: ignore diff --git a/packages/overture-schema/changelog.d/622.misc.md b/packages/overture-schema/changelog.d/622.misc.md new file mode 100644 index 000000000..6f4b755ab --- /dev/null +++ b/packages/overture-schema/changelog.d/622.misc.md @@ -0,0 +1 @@ +Moved `validate()` and `validate_json()` into the new `overture-schema-validation` dependency; the `overture.schema` namespace root is now a bare pkgutil shim. diff --git a/packages/overture-schema/pyproject.toml b/packages/overture-schema/pyproject.toml index 22d67998b..68908307b 100644 --- a/packages/overture-schema/pyproject.toml +++ b/packages/overture-schema/pyproject.toml @@ -9,9 +9,7 @@ dependencies = [ "overture-schema-theme-divisions>=0.1.1", "overture-schema-theme-places>=0.1.1", "overture-schema-theme-transportation>=0.1.1", - "overture-schema-common>=0.1.1", - "pydantic>=2.12.0", - "pyyaml>=6.0.2", + "overture-schema-validation>=0.1.1", "overture-schema-cli>=0.1.1", ] description = "Complete Overture Maps schema collection with all themes and types" @@ -28,24 +26,17 @@ Issues = "https://github.com/OvertureMaps/schema/issues" [tool.uv.sources] overture-schema-cli = { workspace = true } -overture-schema-common = { workspace = true } overture-schema-theme-addresses = { workspace = true } overture-schema-theme-base = { workspace = true } overture-schema-theme-buildings = { workspace = true } overture-schema-theme-divisions = { workspace = true } overture-schema-theme-places = { workspace = true } overture-schema-theme-transportation = { workspace = true } +overture-schema-validation = { workspace = true } [build-system] build-backend = "hatchling.build" requires = ["hatchling"] -[dependency-groups] -dev = [ - "pyyaml>=6.0.2", - "deepdiff>=8.6.0", - "yamlcore>=0.0.4", -] - [tool.hatch.build.targets.wheel] packages = ["src/overture"] diff --git a/packages/overture-schema/src/overture/schema/__init__.py b/packages/overture-schema/src/overture/schema/__init__.py index 3728b389a..8db66d3d0 100644 --- a/packages/overture-schema/src/overture/schema/__init__.py +++ b/packages/overture-schema/src/overture/schema/__init__.py @@ -1,171 +1 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__) - -from collections.abc import Generator -from functools import reduce -from operator import or_ -from types import UnionType -from typing import Annotated, Any, Literal, cast, get_args, get_origin - -from pydantic import BaseModel, Field, Tag, TypeAdapter - -from overture.schema.common import OvertureFeature -from overture.schema.system.discovery import discover_models -from overture.schema.system.feature import Feature - - -def validate(data: object) -> BaseModel: - """ - Validate a Python object, which can be a dictionary or model instance, using the union of all - discovered Overture models. - - Parameters - ---------- - data : object - Python object to validate against the model. - - Returns - ------- - BaseModel - Validated model class - - Raises - ------ - ValidationError - If `data` is not valid according to one of the discovered Overture models - """ - tap = _union_type_adapter() - - return cast(BaseModel, tap.validate_python(data)) - - -def validate_json(json_data: str | bytes | bytearray) -> BaseModel: - """ - Validate JSON data using the union of all discovered Overture models. - - Parameters - ---------- - data : str | bytes | bytearray - JSON data to validate - - Returns - ------- - BaseModel - Validated model class - - Raises - ------ - ValidationError - If `json_data` is not valid according to one of the discovered Overture models - """ - tap = _union_type_adapter() - - return cast(BaseModel, tap.validate_json(json_data)) - - -__all__ = [ - "validate", - "validate_json", -] - - -def _union_type_adapter() -> TypeAdapter: - """ - Return a Pydantic type adapter that can validate the union of all models discovered using entry - points. - """ - models = discover_models() - if not models: - raise RuntimeError("no registered models found via entry points") - - discriminated_models: tuple[type[OvertureFeature], ...] = tuple( - cast(type[OvertureFeature], m) for m in models.values() if _can_discriminate(m) - ) - discriminated_union: UnionType | None = _discriminated_union(discriminated_models) - - non_discriminated_models: Generator[type[BaseModel], None, None] = ( - m for m in models.values() if not _can_discriminate(m) - ) - non_discriminated_union: UnionType | None = reduce( - or_, non_discriminated_models, None - ) - - if discriminated_union and non_discriminated_union: - model_union = discriminated_union | non_discriminated_union - elif discriminated_union: - model_union = discriminated_union - elif non_discriminated_union: - model_union = non_discriminated_union - else: - raise RuntimeError("logic error: unreachable code") - - return TypeAdapter(model_union) - - -def _discriminated_union( - feature_classes: tuple[type[OvertureFeature], ...], -) -> Any: # noqa: ANN401 - """ - Create a discriminated union of the Overture features since they can be discriminated on the - `type` field. This is just a performance optimization, and the union will work even if no models - are discriminated. - """ - if not feature_classes: - return None - else: - return Annotated[ - reduce( - or_, - ( - Annotated[f, Tag(cast(str, _typeliteral(f)))] - for f in feature_classes - ), - ), - Field(discriminator=Feature.field_discriminator("type", *feature_classes)), - ] - - -def _can_discriminate(model_class: object) -> bool: - """ - Return true if given value can participate in a discriminated union on the `type` field because - it is an Overture feature with where the `type` field has a single literal value. - """ - return ( - isinstance(model_class, type) - and issubclass(model_class, OvertureFeature) - and _typeliteral(cast(type[OvertureFeature], model_class)) is not None - ) - - -def _typeliteral(feature_class: type[OvertureFeature]) -> object: - """ - Return the literal value of the Overture Feature model's `type` field, if it has one, or `None` - if it does not. - - Parameters - ---------- - feature_class : type[OvertureFeature] - Overture feature model class - - Returns - ------- - object - The literal constrained value of the model class' `type` field, or `None` if the `type` - field does not have a literal value - - Raises - ------ - TypeError - If the `type` field is constrained to `Literal[None]`, as this is absurd - """ - type_type = feature_class.model_fields["type"].annotation - while get_origin(type_type) is Annotated: - type_type = get_args(Annotated)[0] - if get_origin(type_type) is not Literal: - return None - literal = get_args(type_type)[0] - if literal is None: - raise TypeError( - f"literal value of `type` field for `{OvertureFeature.__name__}` class " - f"`{feature_class.__name__}` is constrained to `None`" - ) - return literal diff --git a/pyproject.toml b/pyproject.toml index 9f5eea2bc..dd83dc5b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ pythonpath = [ "packages/overture-schema-theme-divisions/tests", "packages/overture-schema-theme-places/tests", "packages/overture-schema-theme-transportation/tests", - "packages/overture-schema/tests", + "packages/overture-schema-validation/tests", ] verbosity_subtests = 0 diff --git a/uv.lock b/uv.lock index 4a025e9bc..40975c561 100644 --- a/uv.lock +++ b/uv.lock @@ -26,6 +26,7 @@ members = [ "overture-schema-theme-divisions", "overture-schema-theme-places", "overture-schema-theme-transportation", + "overture-schema-validation", "overture-schema-workspace", ] @@ -896,43 +897,25 @@ version = "1.17.1" source = { editable = "packages/overture-schema" } dependencies = [ { name = "overture-schema-cli" }, - { name = "overture-schema-common" }, { name = "overture-schema-theme-addresses" }, { name = "overture-schema-theme-base" }, { name = "overture-schema-theme-buildings" }, { name = "overture-schema-theme-divisions" }, { name = "overture-schema-theme-places" }, { name = "overture-schema-theme-transportation" }, - { name = "pydantic" }, - { name = "pyyaml" }, -] - -[package.dev-dependencies] -dev = [ - { name = "deepdiff" }, - { name = "pyyaml" }, - { name = "yamlcore" }, + { name = "overture-schema-validation" }, ] [package.metadata] requires-dist = [ { name = "overture-schema-cli", editable = "packages/overture-schema-cli" }, - { name = "overture-schema-common", editable = "packages/overture-schema-common" }, { name = "overture-schema-theme-addresses", editable = "packages/overture-schema-theme-addresses" }, { name = "overture-schema-theme-base", editable = "packages/overture-schema-theme-base" }, { name = "overture-schema-theme-buildings", editable = "packages/overture-schema-theme-buildings" }, { name = "overture-schema-theme-divisions", editable = "packages/overture-schema-theme-divisions" }, { name = "overture-schema-theme-places", editable = "packages/overture-schema-theme-places" }, { name = "overture-schema-theme-transportation", editable = "packages/overture-schema-theme-transportation" }, - { name = "pydantic", specifier = ">=2.12.0" }, - { name = "pyyaml", specifier = ">=6.0.2" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "deepdiff", specifier = ">=8.6.0" }, - { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "yamlcore", specifier = ">=0.0.4" }, + { name = "overture-schema-validation", editable = "packages/overture-schema-validation" }, ] [[package]] @@ -1031,6 +1014,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "deepdiff" }, { name = "jsonpath-ng" }, { name = "types-pyyaml" }, { name = "types-shapely" }, @@ -1045,6 +1029,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "deepdiff", specifier = ">=8.6.0" }, { name = "jsonpath-ng", specifier = ">=1.7.0" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250516" }, { name = "types-shapely", specifier = ">=2.1.0.20250710" }, @@ -1200,6 +1185,35 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.0" }, ] +[[package]] +name = "overture-schema-validation" +version = "0.1.1" +source = { editable = "packages/overture-schema-validation" } +dependencies = [ + { name = "overture-schema-common" }, + { name = "overture-schema-system" }, + { name = "pydantic" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pyyaml" }, + { name = "yamlcore" }, +] + +[package.metadata] +requires-dist = [ + { name = "overture-schema-common", editable = "packages/overture-schema-common" }, + { name = "overture-schema-system", editable = "packages/overture-schema-system" }, + { name = "pydantic", specifier = ">=2.12.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pyyaml", specifier = ">=6.0.2" }, + { name = "yamlcore", specifier = ">=0.0.4" }, +] + [[package]] name = "overture-schema-workspace" version = "0.0.0"