[FEATURE] Add an extension mechanism with discovery and codegen support

README.pydantic.md

@@ -181,7 +181,7 @@ to filter the working set without importing every model:
 from overture.schema.system.discovery import (
     TagSelector,
     discover_models,
-    filter_models,
+    select_models,
 )
 
 models = discover_models()
@@ -193,7 +193,7 @@ models = discover_models()
 #   ...
 # }
 
-buildings = filter_models(
+buildings = select_models(
     models,
     TagSelector(include_any=("overture:theme=buildings",)),
 )
@@ -206,6 +206,63 @@ to attach custom tags during discovery. See the [`overture-schema-system`
 README](packages/overture-schema-system/README.md#tagging) for tag format,
 reserved namespaces, and provider authoring.
 
+### Model Extensions
+
+A data producer can attach optional fields to models it does not own. An
+extension is registered on the same `overture.models` entry-point group as any
+other model; discovery recognizes it as an extension by its `Extends` metadata.
+A model extension declares its targets with `@extends`:
+
+```python
+from overture.schema.system.extension import extends
+
+@extends(Place)
+class OperatingHours(BaseModel):
+    primary: list[str]
+```
+
+Non-model extensions (e.g. a scalar `NewType`) use `Extends(...)` inside
+`Annotated` metadata instead. During discovery each extension is exposed as a
+standalone one-field wrapper model (hidden by `select_models` unless the
+`extension` tag is engaged), and the extension pass adds the field -- optional,
+named after the entry point -- to every registered model the targets resolve
+to.
+
+#### How Targets Resolve
+
+A target may be a model class or a type expression resolving to model classes:
+unions, `Annotated`, `NewType`, and `RootModel`. Two rules govern resolution:
+
+- A union qualifies only if *every* arm resolves to models -- `Place | int` is
+  rejected as a target.
+- A `RootModel` subclass is never a model leaf itself, even though it is a
+  `BaseModel` subclass. It is an alias for its root annotation, and resolution
+  recurses into the root -- at any nesting depth.
+
+The second rule cuts both ways: a `RootModel` over models is an alias for its
+arms, while a `RootModel` over a scalar resolves to no model at all, even when
+nested inside an otherwise valid expression:
+
+```python
+class Segment(RootModel[RoadSegment | RailSegment]):
+    pass
+
+class Version(RootModel[int]):
+    pass
+
+Extends(Segment)            # OK -- extends RoadSegment and RailSegment
+Extends(Version)            # TypeError -- scalar root resolves to no models
+Extends(Segment | Version)  # TypeError -- every union arm must resolve
+```
+
+The extension pass applies the same alias view to registered entries: a
+registered `Segment` is rebuilt as a subclass whose root annotation carries the
+extended arms, while a registered `Version` passes through unchanged, since its
+scalar root contains nothing to extend. Container types (`list[Place]`,
+`dict[str, Place]`) are opaque on both sides: models nested inside them are
+neither valid targets nor rewritten. A self-referential root has no finite
+shape and is rejected.
+
 ## Development
 
 This project uses [uv](https://docs.astral.sh/uv/) for dependency management:

packages/overture-schema-cli/changelog.d/634.feature.md

@@ -0,0 +1 @@
+Listed standalone extension entries in `list-types` while keeping their permissive wrapper models out of validation unless explicitly selected; extension data validates through the feature models it extends.

packages/overture-schema-cli/src/overture/schema/cli/commands.py

@@ -23,7 +23,7 @@
     ModelKey,
     TagSelector,
     discover_models,
-    filter_models,
+    select_models,
 )
 from overture.schema.system.discovery.tag import get_values_for_key
 from overture.schema.system.feature import Feature
@@ -35,7 +35,10 @@
     group_errors_by_discriminator,
     select_most_likely_errors,
 )
-from .tag_options import build_selector, tag_selection_options
+from .tag_options import (
+    build_selector,
+    tag_selection_options,
+)
 from .type_analysis import StructuralTuple, get_item_index, introspect_union
 from .types import ErrorLocation, UnionType, ValidationErrorDict
 
@@ -201,9 +204,15 @@ def resolve_types(
     *,
     type_names: tuple[str, ...] = (),
 ) -> UnionType:
-    """Resolve a TagSelector + type-names into a Pydantic union type."""
+    """Resolve a TagSelector + type-names into a Pydantic union type.
+
+    Uses the discovery layer's default-hidden policy (`select_models`): standalone
+    extension wrapper models are excluded unless the caller engages the ``extension``
+    tag or names a type that only a hidden entry provides. The extension *fields*
+    they contribute remain available on the feature models they target.
+    """
     models = discover_models()
-    models = filter_models(models, selector, type_names=type_names)
+    models = select_models(models, selector, type_names=type_names)
 
     if not models:
         raise ValueError("No models found matching the specified criteria")
@@ -859,7 +868,13 @@ def list_types(
     """
     try:
         models = discover_models()
-        models = filter_models(models, build_selector(tags, filters, excludes))
+        # A listing is introspection, not selection: show every discoverable
+        # entry, extension wrappers included.
+        models = select_models(
+            models,
+            build_selector(tags, filters, excludes),
+            include_extension_entries=True,
+        )
 
         if group_by:
             grouped_models: dict[str, set[ModelKey]] = {}

packages/overture-schema-cli/tests/test_resolve_types.py

@@ -1,6 +1,6 @@
-"""Tests for resolve_types — CLI glue between filter_models and union creation.
+"""Tests for resolve_types — CLI glue between select_models and union creation.
 
-The combinator algebra of filter_models itself is covered in
+The selector combinator algebra itself is covered in
 `test_discovery_filter_models.py` in the system package.
 """
 

packages/overture-schema-codegen/changelog.d/634.feature.md

@@ -0,0 +1 @@
+Added extension-field provenance to extraction (`is_extension`) and rendered an *(extension)* tag on extension-contributed fields in generated markdown.

packages/overture-schema-codegen/src/overture/schema/codegen/cli.py

@@ -6,10 +6,13 @@
 
 import click
 
-from overture.schema.cli.tag_options import build_selector, tag_selection_options
+from overture.schema.cli.tag_options import (
+    build_selector,
+    tag_selection_options,
+)
 from overture.schema.system.discovery import (
     discover_models,
-    filter_models,
+    select_models,
 )
 
 from .extraction.specs import ModelSpec
@@ -57,7 +60,9 @@ def cli() -> None:
 @cli.command("list")
 def list_models() -> None:
     """List all discovered models."""
-    models = discover_models()
+    # A listing is introspection, not selection: show every discoverable
+    # entry, extension wrappers included.
+    models = select_models(discover_models(), include_extension_entries=True)
     names = sorted(
         model.__name__ if isinstance(model, type) else str(model)
         for model in models.values()
@@ -103,7 +108,7 @@ def generate(
 
     all_models = discover_models()
 
-    models = filter_models(all_models, build_selector(tags, filters, excludes))
+    models = select_models(all_models, build_selector(tags, filters, excludes))
 
     if output_dir:
         output_dir.mkdir(parents=True, exist_ok=True)

packages/overture-schema-codegen/src/overture/schema/codegen/extraction/model_extraction.py

@@ -8,6 +8,7 @@
 from pydantic.fields import FieldInfo
 from pydantic_core import PydanticUndefined
 
+from overture.schema.system.extension import applied_extensions
 from overture.schema.system.model_constraint import ModelConstraint
 
 from .docstring import clean_docstring
@@ -153,6 +154,7 @@ def _extract_model_recursive(
     descendant_ancestors = ancestors | {model_class}
 
     model_resolver, union_resolver = _make_resolvers(cache, descendant_ancestors)
+    extensions = applied_extensions(model_class)
 
     fields: list[FieldSpec] = []
     for field_name in _field_order(model_class):
@@ -179,6 +181,7 @@ def _extract_model_recursive(
                 description=field_info.description or ti_description,
                 is_required=_is_field_required(field_info, is_optional),
                 is_optional=is_optional,
+                is_extension=field_name in extensions,
             )
         )
 

packages/overture-schema-codegen/src/overture/schema/codegen/extraction/specs.py

@@ -133,6 +133,7 @@ class FieldSpec:
     description: str | None = None
     is_required: bool = True
     is_optional: bool = False
+    is_extension: bool = False
 
 
 @dataclass

packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_analyzer.py

@@ -51,6 +51,8 @@
 from pydantic.fields import FieldInfo
 from typing_extensions import Sentinel, assert_never, evaluate_forward_ref
 
+from overture.schema.system.extension import Extends
+
 from .docstring import clean_docstring
 from .field import (
     AnyScalar,
@@ -294,18 +296,21 @@ def _recurse(
         return NewTypeShape(name=ctx.name, ref=ctx.ref, inner=inner), opt, desc
 
     if origin is Annotated:
-        args = get_args(annotation)
-        inner_annotation = args[0]
+        inner_annotation, *metadata = get_args(annotation)
         own_desc: str | None = None
         collected: list[ConstraintSource] = []
-        for c in args[1:]:
-            if isinstance(c, FieldInfo):
-                if c.description is not None and own_desc is None:
-                    own_desc = clean_docstring(c.description)
-                for m in c.metadata:
-                    collected.append(_constraint_source(m, newtype_ctx))
+        for item in metadata:
+            constraint_items: tuple[object, ...]
+            if isinstance(item, FieldInfo):
+                if own_desc is None and item.description is not None:
+                    own_desc = clean_docstring(item.description)
+                constraint_items = tuple(item.metadata)
             else:
-                collected.append(_constraint_source(c, newtype_ctx))
+                constraint_items = (item,)
+            for constraint in constraint_items:
+                # `Extends` is extension-target metadata, not a field constraint.
+                if not isinstance(constraint, Extends):
+                    collected.append(_constraint_source(constraint, newtype_ctx))
 
         # Pick the annotation to recurse into and the optionality this
         # Annotated layer contributes. A directly-wrapped union is peeled
@@ -514,10 +519,18 @@ def attach_field_metadata(shape: FieldShape, field_info: FieldInfo) -> FieldShap
     wrapping applies here just as it does during normal annotation
     unwrapping: the constraints anchor at the topmost constraint-bearing
     layer. Returns *shape* unchanged when there is no metadata.
+
+    `Extends` is excluded here as well as in the `Annotated` frame: pydantic
+    hoists a top-level `Annotated`'s metadata into `field_info.metadata`, so
+    extension-target declarations would otherwise re-enter as constraints.
     """
-    if not field_info.metadata:
+    extra = tuple(
+        ConstraintSource(None, None, m)
+        for m in field_info.metadata
+        if not isinstance(m, Extends)
+    )
+    if not extra:
         return shape
-    extra = tuple(ConstraintSource(None, None, m) for m in field_info.metadata)
     return attach_constraints(shape, extra)
 
 

packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py

@@ -97,7 +97,7 @@ def extract_discriminator(
 
 
 _TypeShape = tuple[object, ...]
-_FieldKey = tuple[str, _TypeShape, frozenset[object]]
+_FieldKey = tuple[str, _TypeShape, frozenset[object], bool]
 
 
 def _structural_fingerprint(spec: FieldSpec) -> _TypeShape:
@@ -242,8 +242,15 @@ def extract_union(
             # that already handles a field present on only some arms
             # (`check_builder._field_checks_for_union`), and the renderer's
             # collision resolver already disambiguates multiple `Check`s
-            # landing on the same field label.
-            key = (fs.name, _structural_fingerprint(fs), _constraints_fingerprint(fs))
+            # landing on the same field label. Provenance joins the key so a
+            # native field and an identically-shaped extension field never
+            # collapse into one row.
+            key = (
+                fs.name,
+                _structural_fingerprint(fs),
+                _constraints_fingerprint(fs),
+                fs.is_extension,
+            )
             existing = seen.get(key)
             prior_sources = existing.variant_sources or () if existing else ()
             seen[key] = AnnotatedField(

packages/overture-schema-codegen/src/overture/schema/codegen/markdown/renderer.py

@@ -271,6 +271,11 @@ def _expandable_list_suffix(field_spec: FieldSpec) -> str:
     return "[]" * depth if depth > 0 else ""
 
 
+def _extension_tag(field_spec: FieldSpec) -> str:
+    """Return an italic ` *(extension)*` tag for extension-contributed fields, else ""."""
+    return " *(extension)*" if field_spec.is_extension else ""
+
+
 def _expand_sub_model(
     field_spec: FieldSpec,
     name: str,
@@ -320,7 +325,13 @@ def _expand_model_fields(
     for field_spec in fields:
         row = _field_template_context(field_spec, ctx)
         name = f"{prefix}{field_spec.name}" if prefix else field_spec.name
-        row["name"] = f"{name}{_expandable_list_suffix(field_spec)}"
+        display_name = f"{name}{_expandable_list_suffix(field_spec)}"
+        tag = _extension_tag(field_spec)
+        if tag:
+            row["name"] = f"`{display_name}`{tag}"
+            row["pre_formatted"] = True
+        else:
+            row["name"] = display_name
         if not prefix:
             _annotate_field_constraints(row, field_spec, ctx)
         result.append(row)
@@ -348,10 +359,10 @@ def _short_variant_name(class_name: str, union_name: str) -> str:
     return class_name
 
 
-def _variant_tag(annotated: AnnotatedField, union_name: str) -> str | None:
-    """Return an italic variant tag like `*(Road, Water)*`, or None for shared fields."""
+def _variant_tag(annotated: AnnotatedField, union_name: str) -> str:
+    """Return an italic variant tag like `*(Road, Water)*`, or "" for shared fields."""
     if annotated.variant_sources is None:
-        return None
+        return ""
     short_names = [
         _short_variant_name(v.__name__, union_name) for v in annotated.variant_sources
     ]
@@ -371,20 +382,20 @@ def _expand_union_fields(
     result: list[_FieldRow] = []
     for annotated in spec.annotated_fields:
         field_spec = annotated.field_spec
-        row = _field_template_context(field_spec, ctx)
         name = field_spec.name
-        suffix = _expandable_list_suffix(field_spec)
+        display_name = f"{name}{_expandable_list_suffix(field_spec)}"
+        row = _field_template_context(field_spec, ctx)
 
         _annotate_field_constraints(row, field_spec, ctx)
-        if constraint_notes and field_spec.name in constraint_notes:
-            _annotate_constraint_notes(row, constraint_notes[field_spec.name])
+        if constraint_notes and name in constraint_notes:
+            _annotate_constraint_notes(row, constraint_notes[name])
 
-        tag = _variant_tag(annotated, spec.name)
-        if tag is not None:
-            row["name"] = f"`{name}{suffix}`{tag}"
+        tags = _variant_tag(annotated, spec.name) + _extension_tag(field_spec)
+        if tags:
+            row["name"] = f"`{display_name}`{tags}"
             row["pre_formatted"] = True
         else:
-            row["name"] = f"{name}{suffix}"
+            row["name"] = display_name
 
         result.append(row)
         _expand_sub_model(field_spec, name, ctx, result)

packages/overture-schema-codegen/tests/codegen_test_support.py

@@ -36,7 +36,7 @@
 from overture.schema.system.discovery import (
     TagSelector,
     discover_models,
-    filter_models,
+    select_models,
 )
 from overture.schema.system.discovery.tag import get_values_for_key
 from overture.schema.system.doc import DocumentedEnum
@@ -436,8 +436,10 @@ def flat_specs_from_discovery(
     """Build a flat list of RecordSpecs from discovery, with entry_point set."""
     models = discover_models()
     if theme:
-        models = filter_models(
-            models, TagSelector(include_any=(f"overture:theme={theme}",))
+        models = select_models(
+            models,
+            TagSelector(include_any=(f"overture:theme={theme}",)),
+            include_extension_entries=True,
         )
     return [
         spec