From 9bc187c2ca72935fe069a89c5c9ec693e6adc42e Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 19 Aug 2026 23:47:35 +0200 Subject: [PATCH] feat(substitution): translate universal macros --- README.md | 1 + docs/universal-macro-translation.md | 57 ++++ src/adcp/__init__.py | 32 ++ src/adcp/substitution.py | 242 ++++++++++++++ tests/conformance/substitution/__init__.py | 1 + .../test_universal_macro_translation.py | 160 +++++++++ .../universal-macro-translation/README.md | 12 + .../universal-macro-translation.json | 308 ++++++++++++++++++ .../universal-macro-translation.schema.json | 131 ++++++++ tests/fixtures/public_api_snapshot.json | 9 + 10 files changed, 953 insertions(+) create mode 100644 docs/universal-macro-translation.md create mode 100644 src/adcp/substitution.py create mode 100644 tests/conformance/substitution/__init__.py create mode 100644 tests/conformance/substitution/test_universal_macro_translation.py create mode 100644 tests/conformance/vectors/universal-macro-translation/README.md create mode 100644 tests/conformance/vectors/universal-macro-translation/universal-macro-translation.json create mode 100644 tests/conformance/vectors/universal-macro-translation/universal-macro-translation.schema.json diff --git a/README.md b/README.md index f5ce40f19..b44cdd031 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,7 @@ forward traffic degrades gracefully rather than failing. - **[Migrating from SDK 7 to 8](https://github.com/adcontextprotocol/adcp-client-python/blob/main/MIGRATION_v7_to_v8.md)** - Secure webhook defaults and telemetry changes - **[Migrating from AdCP 3.1 to 3.2 beta](MIGRATION_ADCP_3.1_TO_3.2.md)** - Compact lifecycle adoption and old/new compatibility matrix - **[Testing your AdCP server](docs/testing-your-adcp-server.md)** - In-process harness for unit tests plus storyboard-runner compliance grading +- **[Universal macro translation](docs/universal-macro-translation.md)** - Producer-side pixel URL translation, trust boundary, and diagnostics - **[Multi-tenant contract](docs/multi-tenant-contract.md)** - Scope invariants every multi-tenant agent must satisfy - **[Examples](examples/)** - Code examples and usage patterns diff --git a/docs/universal-macro-translation.md b/docs/universal-macro-translation.md new file mode 100644 index 000000000..7cff4fc0c --- /dev/null +++ b/docs/universal-macro-translation.md @@ -0,0 +1,57 @@ +# Universal macro translation + +Sellers can translate AdCP universal macros in pixel URL query values before +publishing a creative. The helper preserves the path, query keys, fragments, +and literal parameters byte-for-byte while translating only mapped universal +macros in parameter values. + +```python +from adcp.substitution import ( + NativeMacroMapping, + ValueMacroMapping, + translate_universal_macros, +) + +result = translate_universal_macros( + "https://pixel.example/i?buy={MEDIA_BUY_ID}&gdpr={GDPR_CONSENT}", + { + "{MEDIA_BUY_ID}": ValueMacroMapping(value="mb/123"), + "{GDPR_CONSENT}": NativeMacroMapping(native="%%GDPR_CONSENT%%"), + }, +) + +assert result.url == ( + "https://pixel.example/i?buy=mb%2F123&gdpr=%%GDPR_CONSENT%%" +) +``` + +`ValueMacroMapping` is for literal data. It UTF-8 encodes the value and +percent-escapes every byte outside the RFC 3986 unreserved set. Use +`NativeMacroMapping` only for a downstream ad-server token that must be +inserted verbatim. + +## Trust boundary and diagnostics + +Native mappings bypass URL encoding. Before producing any URL, the helper +therefore rejects U+0000–U+001F and U+007F in every native mapping, including +unused entries. Catch `UniversalMacroTranslationError` and inspect its stable +`code` (`unsafe_native_mapping`) and `macro` attributes if this is an expected +configuration boundary. + +Always inspect the result diagnostics before publishing the tracker: + +- `dropped_params` lists each query-parameter occurrence removed because it + contained an unmapped macro. +- `unmapped_macros` lists the missing macros once, in first query occurrence + order. +- `dropped_consent_macros` highlights missing consent mappings. +- `frozen_consent_macros` highlights consent macros supplied as literal + `value` mappings. They are encoded and emitted, but may freeze consent that + should instead be resolved per impression. +- `suspect_native_values` highlights literal values shaped like common native + ad-server tokens and is ordered by mapping insertion order. + +The translator is single-pass: macro-shaped text inside a mapped value is data, +not another substitution. A bare trailing `?` is normalized away. The behavior +is continuously checked against the shared AdCP 3.2 compliance fixture from +[AdCP #6674](https://github.com/adcontextprotocol/adcp/issues/6674). diff --git a/src/adcp/__init__.py b/src/adcp/__init__.py index be666a041..c27c33608 100644 --- a/src/adcp/__init__.py +++ b/src/adcp/__init__.py @@ -180,6 +180,17 @@ def _resolve_version() -> str: "FileCursorStore", "RegistrySync", ), + "adcp.substitution": ( + "MacroMapping", + "MacroMappingEntry", + "NativeMacroMapping", + "TranslateUniversalMacrosResult", + "UniversalMacroTranslationError", + "UniversalMacroTranslationErrorCode", + "ValueMacroMapping", + "encode_unreserved", + "translate_universal_macros", + ), "adcp.testing": ( "CREATIVE_AGENT_CONFIG", "TEST_AGENT_A2A_CONFIG", @@ -828,6 +839,16 @@ def get_adcp_version() -> str: "CursorStore", "FileCursorStore", "ChangeHandler", + # Universal macro substitution + "MacroMapping", + "MacroMappingEntry", + "NativeMacroMapping", + "TranslateUniversalMacrosResult", + "UniversalMacroTranslationError", + "UniversalMacroTranslationErrorCode", + "ValueMacroMapping", + "encode_unreserved", + "translate_universal_macros", # Wholesale feed mirror "FeedMirror", "FeedMirrorClient", @@ -1510,6 +1531,17 @@ def get_adcp_version() -> str: FileCursorStore, RegistrySync, ) + from adcp.substitution import ( + MacroMapping, + MacroMappingEntry, + NativeMacroMapping, + TranslateUniversalMacrosResult, + UniversalMacroTranslationError, + UniversalMacroTranslationErrorCode, + ValueMacroMapping, + encode_unreserved, + translate_universal_macros, + ) from adcp.testing import ( CREATIVE_AGENT_CONFIG, TEST_AGENT_A2A_CONFIG, diff --git a/src/adcp/substitution.py b/src/adcp/substitution.py new file mode 100644 index 000000000..043599ebf --- /dev/null +++ b/src/adcp/substitution.py @@ -0,0 +1,242 @@ +"""Producer-side translation of AdCP universal macros in pixel URLs. + +``native`` mappings are a deliberate raw-token escape hatch for downstream ad +servers. They bypass percent-encoding, so this module validates every native +entry before translating the URL, including entries the URL does not use. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Literal, TypeAlias + +_UNIVERSAL_MACRO = re.compile(r"\{[A-Z][A-Z0-9_]*\}") +_NATIVE_TOKEN_SHAPE = re.compile( + r"(?:%%[^\n\r\u2028\u2029]+%%|" + r"\{\{[^\n\r\u2028\u2029]+\}\}|" + r"\$\{[^\n\r\u2028\u2029]+\}|" + r"\[[A-Z][A-Z0-9_]*\])" +) +_CONSENT_MACROS = frozenset( + { + "{GDPR}", + "{GDPR_CONSENT}", + "{US_PRIVACY}", + "{GPP_STRING}", + "{GPP_SID}", + "{LIMIT_AD_TRACKING}", + } +) +_UNRESERVED_BYTES = frozenset(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~") + + +@dataclass(frozen=True, slots=True) +class NativeMacroMapping: + """A downstream ad-server token inserted without percent-encoding.""" + + native: str + + +@dataclass(frozen=True, slots=True) +class ValueMacroMapping: + """A literal value encoded with the RFC 3986 unreserved whitelist.""" + + value: str + + +MacroMappingEntry: TypeAlias = NativeMacroMapping | ValueMacroMapping +MacroMapping: TypeAlias = Mapping[str, MacroMappingEntry] +UniversalMacroTranslationErrorCode: TypeAlias = Literal["unsafe_native_mapping"] + + +@dataclass(slots=True) +class TranslateUniversalMacrosResult: + """Translated URL and deterministic diagnostics. + + ``dropped_params`` preserves query-parameter occurrence order and may + contain duplicate keys. All macro diagnostic lists are deduplicated. + URL-scoped diagnostics use first query occurrence order; mapping-scoped + diagnostics preserve mapping iteration order. + """ + + url: str + dropped_params: list[str] + unmapped_macros: list[str] + dropped_consent_macros: list[str] + frozen_consent_macros: list[str] + suspect_native_values: list[str] + + +class UniversalMacroTranslationError(ValueError): + """Typed rejection raised before an unsafe native token can be emitted.""" + + code: UniversalMacroTranslationErrorCode + macro: str + + def __init__(self, macro: str) -> None: + self.code = "unsafe_native_mapping" + self.macro = macro + super().__init__(f"native mapping for {macro!r} contains an unsafe control character") + + +def encode_unreserved(raw: str) -> str: + """UTF-8 encode ``raw``, escaping every byte outside RFC 3986 unreserved. + + Percent escapes use uppercase hexadecimal. Unlike + :func:`urllib.parse.quote`, this helper does not preserve ``/``. + """ + + return "".join( + chr(byte) if byte in _UNRESERVED_BYTES else f"%{byte:02X}" for byte in raw.encode("utf-8") + ) + + +def _has_unsafe_native_character(value: str) -> bool: + return any(ord(character) <= 0x1F or ord(character) == 0x7F for character in value) + + +def _append_once(items: list[str], seen: set[str], value: str) -> None: + if value not in seen: + seen.add(value) + items.append(value) + + +def translate_universal_macros( + pixel_url: str, + mapping: MacroMapping, +) -> TranslateUniversalMacrosResult: + """Translate universal macros in query-parameter values. + + ``ValueMacroMapping`` values are UTF-8 percent-encoded with + :func:`encode_unreserved`; ``NativeMacroMapping`` values are inserted + verbatim after the full mapping passes the control-character guard. If a + parameter value contains any unmapped universal macro, that whole parameter + is dropped. Query keys, the path, the fragment, and parameters without + universal macros pass through byte-for-byte. Replacement is single-pass. + + Consent macros supplied through ``ValueMacroMapping`` are translated but + reported in ``frozen_consent_macros`` because freezing impression-time + consent at producer time can create a privacy defect. Callers should also + inspect ``dropped_consent_macros`` and ``suspect_native_values`` before + publishing a tracker. + + Raises: + UniversalMacroTranslationError: A native mapping, used or unused, + contains U+0000-U+001F or U+007F. No URL is emitted. + TypeError: A mapping entry is not a supported typed mapping model. + """ + + frozen_consent_macros: list[str] = [] + frozen_seen: set[str] = set() + suspect_native_values: list[str] = [] + suspect_seen: set[str] = set() + validated_mapping: dict[str, MacroMappingEntry] = {} + + # Validate the entire raw-token trust boundary before doing any URL work. + # An unused unsafe entry must reject just like an entry present in the URL. + for macro, entry in mapping.items(): + if isinstance(entry, NativeMacroMapping): + native = entry.native + if _has_unsafe_native_character(native): + raise UniversalMacroTranslationError(macro) + validated_mapping[macro] = NativeMacroMapping(native=native) + elif isinstance(entry, ValueMacroMapping): + value = entry.value + if macro in _CONSENT_MACROS: + _append_once(frozen_consent_macros, frozen_seen, macro) + if _NATIVE_TOKEN_SHAPE.fullmatch(value): + _append_once(suspect_native_values, suspect_seen, macro) + validated_mapping[macro] = ValueMacroMapping(value=value) + else: + raise TypeError( + f"mapping entry for {macro!r} must be NativeMacroMapping " "or ValueMacroMapping" + ) + + fragment_index = pixel_url.find("#") + if fragment_index == -1: + without_fragment = pixel_url + fragment = "" + else: + without_fragment = pixel_url[:fragment_index] + fragment = pixel_url[fragment_index:] + + query_index = without_fragment.find("?") + if query_index == -1: + return TranslateUniversalMacrosResult( + url=pixel_url, + dropped_params=[], + unmapped_macros=[], + dropped_consent_macros=[], + frozen_consent_macros=frozen_consent_macros, + suspect_native_values=suspect_native_values, + ) + + base = without_fragment[:query_index] + raw_query = without_fragment[query_index + 1 :] + + dropped_params: list[str] = [] + unmapped_macros: list[str] = [] + unmapped_seen: set[str] = set() + dropped_consent_macros: list[str] = [] + dropped_consent_seen: set[str] = set() + output_parts: list[str] = [] + + for raw_param in raw_query.split("&"): + key, separator, value = raw_param.partition("=") + tokens = _UNIVERSAL_MACRO.findall(value) + if not tokens: + output_parts.append(raw_param) + continue + + missing = [token for token in tokens if token not in validated_mapping] + if missing: + dropped_params.append(key) + for macro in missing: + _append_once(unmapped_macros, unmapped_seen, macro) + if macro in _CONSENT_MACROS: + _append_once( + dropped_consent_macros, + dropped_consent_seen, + macro, + ) + continue + + def replace(match: re.Match[str]) -> str: + macro = match.group(0) + entry = validated_mapping[macro] + if isinstance(entry, NativeMacroMapping): + return entry.native + if isinstance(entry, ValueMacroMapping): + return encode_unreserved(entry.value) + # The mapping-wide validation above makes this unreachable even + # for mutable custom Mapping implementations under normal use. + raise TypeError(f"unsupported mapping entry for {macro!r}") + + translated = _UNIVERSAL_MACRO.sub(replace, value) + output_parts.append(f"{key}{separator}{translated}") + + new_query = "&".join(output_parts) + url = f"{base}?{new_query}{fragment}" if new_query else f"{base}{fragment}" + return TranslateUniversalMacrosResult( + url=url, + dropped_params=dropped_params, + unmapped_macros=unmapped_macros, + dropped_consent_macros=dropped_consent_macros, + frozen_consent_macros=frozen_consent_macros, + suspect_native_values=suspect_native_values, + ) + + +__all__ = [ + "MacroMapping", + "MacroMappingEntry", + "NativeMacroMapping", + "TranslateUniversalMacrosResult", + "UniversalMacroTranslationError", + "UniversalMacroTranslationErrorCode", + "ValueMacroMapping", + "encode_unreserved", + "translate_universal_macros", +] diff --git a/tests/conformance/substitution/__init__.py b/tests/conformance/substitution/__init__.py new file mode 100644 index 000000000..bec9ef94f --- /dev/null +++ b/tests/conformance/substitution/__init__.py @@ -0,0 +1 @@ +"""Universal-macro substitution conformance tests.""" diff --git a/tests/conformance/substitution/test_universal_macro_translation.py b/tests/conformance/substitution/test_universal_macro_translation.py new file mode 100644 index 000000000..2cc81ca97 --- /dev/null +++ b/tests/conformance/substitution/test_universal_macro_translation.py @@ -0,0 +1,160 @@ +"""Execute the canonical AdCP universal-macro translation fixture.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterator, Mapping +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest +from jsonschema import Draft7Validator + +import adcp +from adcp.substitution import ( + MacroMappingEntry, + NativeMacroMapping, + TranslateUniversalMacrosResult, + UniversalMacroTranslationError, + ValueMacroMapping, + encode_unreserved, + translate_universal_macros, +) + +_VECTORS_DIR = Path(__file__).parent.parent / "vectors" / "universal-macro-translation" +_FIXTURE_PATH = _VECTORS_DIR / "universal-macro-translation.json" +_SCHEMA_PATH = _VECTORS_DIR / "universal-macro-translation.schema.json" +_PINNED_SHA256 = { + _FIXTURE_PATH.name: "f6c767a616b3564d6d96f035f396f35422d909f3506630d07dfea7c4575eeee4", + _SCHEMA_PATH.name: "662e1bb8d7b324f22ef8f56d0729e32be67438c3bd3a35d0dd72b05400c3b08e", +} + + +def _load_json(path: Path) -> dict[str, Any]: + document = json.loads(path.read_text()) + assert isinstance(document, dict) + return document + + +_FIXTURE = _load_json(_FIXTURE_PATH) + + +def _typed_mapping(raw: dict[str, dict[str, str]]) -> dict[str, MacroMappingEntry]: + return { + macro: ( + NativeMacroMapping(native=entry["native"]) + if "native" in entry + else ValueMacroMapping(value=entry["value"]) + ) + for macro, entry in raw.items() + } + + +def test_vendored_fixture_is_exactly_pinned() -> None: + for path in (_FIXTURE_PATH, _SCHEMA_PATH): + assert hashlib.sha256(path.read_bytes()).hexdigest() == _PINNED_SHA256[path.name] + + +def test_vendored_fixture_matches_its_canonical_schema() -> None: + schema = _load_json(_SCHEMA_PATH) + Draft7Validator.check_schema(schema) + Draft7Validator(schema).validate(_FIXTURE) + + +@pytest.mark.parametrize( + "vector", + _FIXTURE["vectors"], + ids=[vector["name"] for vector in _FIXTURE["vectors"]], +) +def test_canonical_universal_macro_translation_vector(vector: dict[str, Any]) -> None: + mapping = _typed_mapping(vector["mapping"]) + if "expected_error" in vector: + with pytest.raises(UniversalMacroTranslationError) as exc_info: + translate_universal_macros(vector["input_pixel_url"], mapping) + assert exc_info.value.code == vector["expected_error"]["code"] + assert exc_info.value.macro == vector["expected_error"]["macro"] + return + + result = translate_universal_macros( + vector["input_pixel_url"], + mapping, + ) + assert asdict(result) == vector["expected"] + + +def test_unsafe_unused_native_mapping_rejects_before_emitting_url() -> None: + with pytest.raises(UniversalMacroTranslationError) as exc_info: + translate_universal_macros( + "https://pixel.example/i?ok=1", + {"{UNUSED}": NativeMacroMapping(native="bad\x00token")}, + ) + assert exc_info.value.code == "unsafe_native_mapping" + assert exc_info.value.macro == "{UNUSED}" + + +@pytest.mark.parametrize( + "codepoint", + [*range(0x20), 0x7F], + ids=lambda codepoint: f"U+{codepoint:04X}", +) +def test_native_mapping_rejects_every_forbidden_control_character(codepoint: int) -> None: + with pytest.raises(UniversalMacroTranslationError) as exc_info: + translate_universal_macros( + "https://pixel.example/i?cb={CACHEBUSTER}", + {"{CACHEBUSTER}": NativeMacroMapping(native=f"before{chr(codepoint)}after")}, + ) + assert exc_info.value.code == "unsafe_native_mapping" + assert exc_info.value.macro == "{CACHEBUSTER}" + + +@pytest.mark.parametrize( + "codepoint", + [*range(0x80, 0xA0), 0x2028, 0x2029], + ids=lambda codepoint: f"U+{codepoint:04X}", +) +def test_native_mapping_does_not_reject_c1_or_unicode_line_separators(codepoint: int) -> None: + native = f"before{chr(codepoint)}after" + result = translate_universal_macros( + "https://pixel.example/i?cb={CACHEBUSTER}", + {"{CACHEBUSTER}": NativeMacroMapping(native=native)}, + ) + assert result.url == f"https://pixel.example/i?cb={native}" + + +def test_mapping_is_not_reread_after_native_values_are_validated() -> None: + class ChangingMapping(Mapping[str, MacroMappingEntry]): + def __init__(self) -> None: + self.reads = 0 + + def __getitem__(self, key: str) -> MacroMappingEntry: + if key != "{CACHEBUSTER}": + raise KeyError(key) + self.reads += 1 + if self.reads == 1: + return NativeMacroMapping(native="%%SAFE%%") + return NativeMacroMapping(native="%%X%%\r\nInjected: yes") + + def __iter__(self) -> Iterator[str]: + return iter(("{CACHEBUSTER}",)) + + def __len__(self) -> int: + return 1 + + mapping = ChangingMapping() + result = translate_universal_macros( + "https://pixel.example/i?cb={CACHEBUSTER}", + mapping, + ) + assert result.url == "https://pixel.example/i?cb=%%SAFE%%" + assert mapping.reads == 1 + + +def test_public_exports_resolve_to_substitution_models_and_helpers() -> None: + assert adcp.NativeMacroMapping is NativeMacroMapping + assert adcp.ValueMacroMapping is ValueMacroMapping + assert adcp.TranslateUniversalMacrosResult is TranslateUniversalMacrosResult + assert adcp.UniversalMacroTranslationError is UniversalMacroTranslationError + assert adcp.encode_unreserved is encode_unreserved + assert adcp.translate_universal_macros is translate_universal_macros diff --git a/tests/conformance/vectors/universal-macro-translation/README.md b/tests/conformance/vectors/universal-macro-translation/README.md new file mode 100644 index 000000000..2a07603fd --- /dev/null +++ b/tests/conformance/vectors/universal-macro-translation/README.md @@ -0,0 +1,12 @@ +# Universal macro translation vectors + +These files are vendored verbatim from the AdCP compliance source at commit +[`acc022a53fad8ecab877d374df1760fef756325f`](https://github.com/adcontextprotocol/adcp/commit/acc022a53fad8ecab877d374df1760fef756325f): + +- `static/compliance/source/test-vectors/universal-macro-translation.json` +- `static/compliance/source/test-vectors/universal-macro-translation.schema.json` + +The test suite pins their SHA-256 digests and executes the fixture directly +against `adcp.substitution.translate_universal_macros`. Update the source +commit, both files, and both digests together when deliberately adopting a new +fixture revision. Do not edit or add expected results in Python. diff --git a/tests/conformance/vectors/universal-macro-translation/universal-macro-translation.json b/tests/conformance/vectors/universal-macro-translation/universal-macro-translation.json new file mode 100644 index 000000000..824f2248f --- /dev/null +++ b/tests/conformance/vectors/universal-macro-translation/universal-macro-translation.json @@ -0,0 +1,308 @@ +{ + "$schema": "./universal-macro-translation.schema.json", + "$schema_version": "1.0", + "version": "3.2", + "name": "Universal macro translation — cross-SDK conformance fixtures", + "spec_reference": "docs/creative/universal-macros.mdx#implementing-translation-with-the-sdk", + "description": "Language-neutral golden vectors for ratified producer-side universal-macro translation behavior, including typed rejection cases and the complete successful result shape.", + "ordering": { + "dropped_params": "Query-parameter occurrence order; repeated keys remain repeated.", + "unmapped_macros": "First occurrence in query order, deduplicated.", + "dropped_consent_macros": "First occurrence in query order, deduplicated subset of unmapped_macros.", + "frozen_consent_macros": "Mapping property order, deduplicated consent macros supplied through value entries.", + "suspect_native_values": "Mapping property order, deduplicated. JSON fixture consumers MUST preserve the mapping order shown here." + }, + "vectors": [ + { + "name": "value-reserved-characters", + "input_pixel_url": "https://pixel.example/i?v={VALUE}", + "mapping": { "{VALUE}": { "value": "/?&=% {}#" } }, + "expected": { + "url": "https://pixel.example/i?v=%2F%3F%26%3D%25%20%7B%7D%23", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "value-unreserved-characters", + "input_pixel_url": "https://pixel.example/i?v={VALUE}", + "mapping": { "{VALUE}": { "value": "AZaz09-._~" } }, + "expected": { + "url": "https://pixel.example/i?v=AZaz09-._~", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "value-non-ascii-utf8", + "input_pixel_url": "https://pixel.example/i?store={STORE_ID}", + "mapping": { "{STORE_ID}": { "value": "café" } }, + "expected": { + "url": "https://pixel.example/i?store=caf%C3%A9", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "native-token-forms-inserted-verbatim", + "input_pixel_url": "https://pixel.example/i?a={A}&b={B}&c={C}&d={D}", + "mapping": { + "{A}": { "native": "%%TOKEN%%" }, + "{B}": { "native": "{{token}}" }, + "{C}": { "native": "${token}" }, + "{D}": { "native": "[TOKEN]" } + }, + "expected": { + "url": "https://pixel.example/i?a=%%TOKEN%%&b={{token}}&c=${token}&d=[TOKEN]", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "multiple-mapped-macros-in-one-parameter", + "input_pixel_url": "https://pixel.example/i?ids=pre-{A}-{B}-{A}", + "mapping": { + "{A}": { "value": "a/b" }, + "{B}": { "native": "%%B%%" } + }, + "expected": { + "url": "https://pixel.example/i?ids=pre-a%2Fb-%%B%%-a%2Fb", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "mapped-plus-unmapped-drops-whole-parameter", + "input_pixel_url": "https://pixel.example/i?ids={MEDIA_BUY_ID}-{UNKNOWN}&literal=keep", + "mapping": { "{MEDIA_BUY_ID}": { "value": "mb_1" } }, + "expected": { + "url": "https://pixel.example/i?literal=keep", + "dropped_params": ["ids"], + "unmapped_macros": ["{UNKNOWN}"], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "repeated-keys-and-first-seen-report-order", + "input_pixel_url": "https://pixel.example/i?first={Z}&same={A}&same={Z}&mix={B}{A}&ok=1", + "mapping": {}, + "expected": { + "url": "https://pixel.example/i?ok=1", + "dropped_params": ["first", "same", "same", "mix"], + "unmapped_macros": ["{Z}", "{A}", "{B}"], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "privacy-drops-and-deduplicated-order", + "input_pixel_url": "https://pixel.example/i?g={GPP_STRING}&u={US_PRIVACY}&g2={GPP_STRING}&c={GDPR_CONSENT}", + "mapping": {}, + "expected": { + "url": "https://pixel.example/i", + "dropped_params": ["g", "u", "g2", "c"], + "unmapped_macros": ["{GPP_STRING}", "{US_PRIVACY}", "{GDPR_CONSENT}"], + "dropped_consent_macros": ["{GPP_STRING}", "{US_PRIVACY}", "{GDPR_CONSENT}"], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "privacy-native-mappings-allowed", + "input_pixel_url": "https://pixel.example/i?g={GPP_STRING}&u={US_PRIVACY}&c={GDPR_CONSENT}", + "mapping": { + "{GPP_STRING}": { "native": "%%GPP_STRING%%" }, + "{US_PRIVACY}": { "native": "%%US_PRIVACY%%" }, + "{GDPR_CONSENT}": { "native": "%%GDPR_CONSENT%%" } + }, + "expected": { + "url": "https://pixel.example/i?g=%%GPP_STRING%%&u=%%US_PRIVACY%%&c=%%GDPR_CONSENT%%", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "literal-parameters-pass-through-byte-for-byte", + "input_pixel_url": "https://pixel.example/i?encoded=%2f+%7e&flag&empty=&equals=a=b", + "mapping": {}, + "expected": { + "url": "https://pixel.example/i?encoded=%2f+%7e&flag&empty=&equals=a=b", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "path-and-no-query-left-untouched", + "input_pixel_url": "https://pixel.example/{MEDIA_BUY_ID}", + "mapping": { "{MEDIA_BUY_ID}": { "value": "mb_1" } }, + "expected": { + "url": "https://pixel.example/{MEDIA_BUY_ID}", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "fragment-left-untouched", + "input_pixel_url": "https://pixel.example/i?buy={MEDIA_BUY_ID}#frag={CREATIVE_ID}", + "mapping": { + "{MEDIA_BUY_ID}": { "value": "mb_1" }, + "{CREATIVE_ID}": { "value": "cr_1" } + }, + "expected": { + "url": "https://pixel.example/i?buy=mb_1#frag={CREATIVE_ID}", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "macro-in-key-left-untouched", + "input_pixel_url": "https://pixel.example/i?{MEDIA_BUY_ID}=literal", + "mapping": { "{MEDIA_BUY_ID}": { "value": "mb_1" } }, + "expected": { + "url": "https://pixel.example/i?{MEDIA_BUY_ID}=literal", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "substitution-is-single-pass", + "input_pixel_url": "https://pixel.example/i?buy={MEDIA_BUY_ID}", + "mapping": { + "{MEDIA_BUY_ID}": { "value": "{PACKAGE_ID}" }, + "{PACKAGE_ID}": { "value": "pkg_1" } + }, + "expected": { + "url": "https://pixel.example/i?buy=%7BPACKAGE_ID%7D", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "suspect-native-values-are-mapping-scoped-and-ordered", + "input_pixel_url": "https://pixel.example/i?ok={NORMAL}", + "mapping": { + "{PERCENT}": { "value": "%%TOKEN%%" }, + "{DOUBLE_BRACE}": { "value": "{{token}}" }, + "{DOLLAR_BRACE}": { "value": "${token}" }, + "{BRACKET}": { "value": "[TOKEN]" }, + "{NORMAL}": { "value": "ok" } + }, + "expected": { + "url": "https://pixel.example/i?ok=ok", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": ["{PERCENT}", "{DOUBLE_BRACE}", "{DOLLAR_BRACE}", "{BRACKET}"] + } + }, + { + "name": "ordinary-bracketed-values-not-suspect", + "input_pixel_url": "https://pixel.example/i?a={A}&b={B}", + "mapping": { + "{A}": { "value": "[1,2,3]" }, + "{B}": { "value": "[redacted]" } + }, + "expected": { + "url": "https://pixel.example/i?a=%5B1%2C2%2C3%5D&b=%5Bredacted%5D", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "privacy-value-mappings-are-encoded-and-reported", + "input_pixel_url": "https://pixel.example/i?g={GPP_STRING}&u={US_PRIVACY}", + "mapping": { + "{GPP_STRING}": { "value": "DBABMA~CPXxRfA/PXxRfA" }, + "{US_PRIVACY}": { "value": "1YNN" } + }, + "expected": { + "url": "https://pixel.example/i?g=DBABMA~CPXxRfA%2FPXxRfA&u=1YNN", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": ["{GPP_STRING}", "{US_PRIVACY}"], + "suspect_native_values": [] + } + }, + { + "name": "bare-query-normalized-away", + "input_pixel_url": "https://pixel.example/i?", + "mapping": {}, + "expected": { + "url": "https://pixel.example/i", + "dropped_params": [], + "unmapped_macros": [], + "dropped_consent_macros": [], + "frozen_consent_macros": [], + "suspect_native_values": [] + } + }, + { + "name": "native-c0-null-rejected", + "input_pixel_url": "https://pixel.example/i?cb={CACHEBUSTER}", + "mapping": { "{CACHEBUSTER}": { "native": "%%CACHE\u0000BUSTER%%" } }, + "expected_error": { + "code": "unsafe_native_mapping", + "macro": "{CACHEBUSTER}" + } + }, + { + "name": "native-c0-unit-separator-rejected", + "input_pixel_url": "https://pixel.example/i?cb={CACHEBUSTER}", + "mapping": { "{CACHEBUSTER}": { "native": "%%CACHE\u001fBUSTER%%" } }, + "expected_error": { + "code": "unsafe_native_mapping", + "macro": "{CACHEBUSTER}" + } + }, + { + "name": "native-del-rejected", + "input_pixel_url": "https://pixel.example/i?cb={CACHEBUSTER}", + "mapping": { "{CACHEBUSTER}": { "native": "%%CACHE\u007fBUSTER%%" } }, + "expected_error": { + "code": "unsafe_native_mapping", + "macro": "{CACHEBUSTER}" + } + } + ] +} diff --git a/tests/conformance/vectors/universal-macro-translation/universal-macro-translation.schema.json b/tests/conformance/vectors/universal-macro-translation/universal-macro-translation.schema.json new file mode 100644 index 000000000..d0d7c8ce5 --- /dev/null +++ b/tests/conformance/vectors/universal-macro-translation/universal-macro-translation.schema.json @@ -0,0 +1,131 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "universal-macro-translation.schema.json", + "title": "Universal macro translation fixture", + "type": "object", + "additionalProperties": false, + "required": [ + "$schema", + "$schema_version", + "version", + "name", + "spec_reference", + "description", + "ordering", + "vectors" + ], + "properties": { + "$schema": { "type": "string" }, + "$schema_version": { "const": "1.0" }, + "version": { "const": "3.2" }, + "name": { "type": "string", "minLength": 1 }, + "spec_reference": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "ordering": { + "type": "object", + "additionalProperties": false, + "required": [ + "dropped_params", + "unmapped_macros", + "dropped_consent_macros", + "frozen_consent_macros", + "suspect_native_values" + ], + "properties": { + "dropped_params": { "type": "string", "minLength": 1 }, + "unmapped_macros": { "type": "string", "minLength": 1 }, + "dropped_consent_macros": { "type": "string", "minLength": 1 }, + "frozen_consent_macros": { "type": "string", "minLength": 1 }, + "suspect_native_values": { "type": "string", "minLength": 1 } + } + }, + "vectors": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/vector" } + } + }, + "definitions": { + "macro": { + "type": "string", + "pattern": "^\\{[A-Z][A-Z0-9_]*\\}$" + }, + "mappingEntry": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["native"], + "properties": { "native": { "type": "string" } } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["value"], + "properties": { "value": { "type": "string" } } + } + ] + }, + "mapping": { + "type": "object", + "propertyNames": { "$ref": "#/definitions/macro" }, + "additionalProperties": { "$ref": "#/definitions/mappingEntry" } + }, + "macroArray": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/definitions/macro" } + }, + "expected": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "dropped_params", + "unmapped_macros", + "dropped_consent_macros", + "frozen_consent_macros", + "suspect_native_values" + ], + "properties": { + "url": { "type": "string" }, + "dropped_params": { + "type": "array", + "items": { "type": "string" } + }, + "unmapped_macros": { "$ref": "#/definitions/macroArray" }, + "dropped_consent_macros": { "$ref": "#/definitions/macroArray" }, + "frozen_consent_macros": { "$ref": "#/definitions/macroArray" }, + "suspect_native_values": { "$ref": "#/definitions/macroArray" } + } + }, + "expectedError": { + "type": "object", + "additionalProperties": false, + "required": ["code", "macro"], + "properties": { + "code": { "const": "unsafe_native_mapping" }, + "macro": { "$ref": "#/definitions/macro" } + } + }, + "vector": { + "type": "object", + "additionalProperties": false, + "required": ["name", "input_pixel_url", "mapping"], + "oneOf": [ + { "required": ["expected"] }, + { "required": ["expected_error"] } + ], + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" + }, + "input_pixel_url": { "type": "string", "minLength": 1 }, + "mapping": { "$ref": "#/definitions/mapping" }, + "expected": { "$ref": "#/definitions/expected" }, + "expected_error": { "$ref": "#/definitions/expectedError" } + } + } + } +} diff --git a/tests/fixtures/public_api_snapshot.json b/tests/fixtures/public_api_snapshot.json index 8a98ce725..63a3880c9 100644 --- a/tests/fixtures/public_api_snapshot.json +++ b/tests/fixtures/public_api_snapshot.json @@ -300,6 +300,8 @@ "LogEventResponse", "LogEventResponse1", "LogEventSuccessResponse", + "MacroMapping", + "MacroMappingEntry", "MarkdownAsset", "McpWebhookPayload", "MediaBuy", @@ -310,6 +312,7 @@ "MediaChannel", "Member", "MemoryBackend", + "NativeMacroMapping", "NotificationConfig", "OfferingAssetConstraint", "OfferingAssetGroup", @@ -453,6 +456,9 @@ "TimeBasedPricingOption", "TimeUnit", "Transform", + "TranslateUniversalMacrosResult", + "UniversalMacroTranslationError", + "UniversalMacroTranslationErrorCode", "UnknownFieldPolicy", "UpdateContentStandardsErrorResponse", "UpdateContentStandardsResponse1", @@ -482,6 +488,7 @@ "ValidationMode", "ValidationOutcome", "ValidationResult", + "ValueMacroMapping", "VastAsset", "VastTrackerAsset", "VcpmAuctionPricingOption", @@ -526,6 +533,7 @@ "creative_agent", "detect_publisher_properties_divergence", "domain_matches", + "encode_unreserved", "extract_webhook_result_data", "fetch_adagents", "fetch_adagents_with_cache", @@ -560,6 +568,7 @@ "test_agent_client", "test_agent_no_auth", "to_wire_dict", + "translate_universal_macros", "upgrade_legacy_format_id", "uses_deprecated_assets_field", "validate_adagents",