From caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466 Mon Sep 17 00:00:00 2001 From: "tyler.potter" Date: Fri, 23 Oct 2026 16:11:41 +0000 Subject: [PATCH] feat: add evaluation case applicability controls Define rules-based platform filters and self-contained feature suites so consumers only run and load supported evaluation coverage. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .../workflows/downstream-compatibility.yml | 1 + .github/workflows/validate-fixtures.yml | 1 + CONTRIBUTING.md | 22 ++- README.md | 70 ++++++-- ci/test_validate_fixtures.py | 113 ++++++++++++- ci/validate-fixtures.py | 159 +++++++++++++++--- evaluation-cases/test-case-disabled-flag.json | 18 ++ evaluation-cases/test-case-empty-flag.json | 18 ++ .../test-case-experimentation-opt-out.json | 18 ++ 9 files changed, 376 insertions(+), 44 deletions(-) diff --git a/.github/workflows/downstream-compatibility.yml b/.github/workflows/downstream-compatibility.yml index 3b25ddfb..39c9a71a 100644 --- a/.github/workflows/downstream-compatibility.yml +++ b/.github/workflows/downstream-compatibility.yml @@ -6,6 +6,7 @@ on: - .github/workflows/downstream-compatibility.yml - ci/run-downstream-conformance.sh - evaluation-cases/** + - evaluation-suites/** - ufc-config.json permissions: diff --git a/.github/workflows/validate-fixtures.yml b/.github/workflows/validate-fixtures.yml index 9f887213..57ce54c0 100644 --- a/.github/workflows/validate-fixtures.yml +++ b/.github/workflows/validate-fixtures.yml @@ -7,6 +7,7 @@ on: - ci/validate-fixtures.py - ci/test_validate_fixtures.py - evaluation-cases/** + - evaluation-suites/** - ufc-config.json permissions: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b5bda6fe..a3af4038 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,6 +52,17 @@ projects ship support. 3. If your test case requires a new flag, add the flag definition to `ufc-config.json` 4. Keep the fixture SDK-neutral. Do not include SDK-specific fields such as `variant` or `flagMetadata`; downstream SDKs should derive those from `ufc-config.json` when they need them. +Use `_skip` or `_include` only for a known cross-platform applicability gap. +Never specify both. Entries must use a canonical identifier from +`ci/validate-fixtures.py` and should include a concise reason. These controls +apply only to rules-based evaluation cases. Do not edit precomputed-assignment +fixtures, schemas, documentation, or validators when changing this contract. + +When a new feature's UFC syntax itself is not supported by every SDK, create a +self-contained `evaluation-suites//` directory containing its own +`ufc-config.json` and `cases/`. Do not add that syntax to the legacy root config: +a suite the consumer has not enabled must not have its configuration parsed. + For selected split/allocation metadata, add optional `result.serialId` (integer or `null`) and `result.doLog` (boolean) expectations. A serial ID of `0` is valid; `null` asserts absence, while omitting the field leaves it unspecified. Keep @@ -106,11 +117,14 @@ revision in downstream repos. Each rules-based consumer should: 2. Automate fixture updates. For submodules, configure Dependabot with `package-ecosystem: "gitsubmodule"`; for snapshots, automate regeneration from the source revision. -3. Load `ufc-config.json` and loop over every `evaluation-cases/*.json` file in - unit tests. -4. Run the cases for every implemented capability as well as the baseline; +3. Treat `ufc-config.json` and `evaluation-cases/` as the legacy `core` suite. +4. Implement `_skip`/`_include` filtering before parametrization and side effects, + using the canonical platform identifiers from `ci/validate-fixtures.py`. +5. Run applicable cases for every implemented capability as well as the baseline; follow the [runner requirements](README.md#implement-the-rules-based-runner). -5. Preserve shared expectations unchanged. Avoid hand-maintained copies and +6. Explicitly opt into supported `evaluation-suites//` directories; do + not auto-enable new suites. +7. Preserve shared expectations unchanged. Avoid hand-maintained copies and programmatic-only replacements for shared evaluator cases. Precomputed-assignment consumers use the same pinning and provenance rules with diff --git a/README.md b/README.md index 6c73cb08..3167b562 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ ffe-system-test-data/ ├── ufc-config.json # Shared rules-based configuration, including malformed test inputs ├── evaluation-cases/ │ └── test-*.json # Evaluation test case files -├── evaluation-suites/ # Self-contained, explicitly activated feature suites +├── evaluation-suites/ # Optional self-contained, explicitly activated feature suites │ └── dependent-flags/ │ ├── ufc-config.json # UFC containing only dependent-flag fixtures │ └── cases/ # Evaluation cases for this feature @@ -121,22 +121,47 @@ results in a downstream copy. ### Implement the rules-based runner -1. Load `ufc-config.json` through the consumer's real configuration parser and - initialize its evaluator. Preserve the malformed inputs used by isolation tests. -2. Discover every `evaluation-cases/*.json` file and parse its array. Keep case +1. Choose one canonical platform identifier from `ci/validate-fixtures.py`. +2. Select suites without loading configuration. `core` is selected by default; + feature suites are explicitly enabled by the consuming repository. +3. Discover the selected suite's case files and parse each array. Keep case + order within each file, including repeated evaluations. Fail if a selected + fixture directory is missing or empty. +4. Apply `_skip`/`_include` to each case for the runner's canonical platform + before test parametrization, setup, evaluation, telemetry expectations, + counts, or any other side effect. An excluded case must not execute and must + not influence drain markers or expected event counts. +5. If no applicable cases remain in a suite, report it as fully excluded and do + not parse, send, or install that suite's UFC configuration. +6. For a suite with applicable cases, load its adjacent `ufc-config.json` + through the consumer's real configuration parser and initialize its evaluator. + Preserve malformed inputs used by isolation tests. +7. Run every remaining baseline case with default provider options, using + `flag`, `variationType`, `defaultValue`, `targetingKey`, and `attributes` as + supplied. Assert the expected value and reason, and any specified error code. + Consumers exposing selected metadata must also check supplied `serialId` and + `doLog` expectations against the actual selection. +8. For each capability the consumer implements, map its name to the native + option and rerun every applicable case carrying that capability. Assert its + expected result instead of the baseline result. Run capabilities separately. +9. Report the fixture SHA and executed and excluded case counts, including + counts per supported capability. Fail if a supported capability discovers no + applicable cases. Excluding every capability case must not look like + successful capability support. + +The legacy `core` suite uses root `ufc-config.json` and `evaluation-cases/`. +Feature suites use `evaluation-suites//ufc-config.json` and the adjacent +`cases/` directory. Never parse the configuration of an unselected suite. + +For backward-compatible core-only consumers, the equivalent minimum flow is: + +1. Discover every `evaluation-cases/*.json` file and parse its array. Keep case order within each file, including repeated evaluations. Fail if the fixture directory is missing or empty. -3. Run every baseline case with default provider options, using `flag`, - `variationType`, `defaultValue`, `targetingKey`, and `attributes` as supplied. - Assert the expected value and reason, and any specified error code. Consumers - exposing selected metadata must also check the supplied `serialId` and `doLog` - expectations against the actual selection. -4. For each capability the consumer implements, map its name to the native - option and rerun every case carrying that capability. Assert its expected - result instead of the baseline result. Run capabilities separately. -5. Report the fixture SHA and executed case counts, including counts per - supported capability. Fail if a supported capability discovers no cases. - Skipping every capability case must not look like successful capability support. +2. Apply the case filters. If none remain, report `core` as fully excluded and + stop without loading configuration. +3. Load root `ufc-config.json`, then run the applicable cases using the remaining + steps above. The adapter may translate API names and native result containers. It must preserve the meaning and precision of the inputs and expected results, including @@ -199,6 +224,8 @@ Each evaluation case uses a universal schema with the following fields: | `result.doLog` | boolean | Optional expected selected allocation's logging decision | | `result.features` | object | Optional exact feature maps for `HOOK`, `EXPOSURE`, and `EVALUATION`; empty maps assert no features, omission leaves routing unspecified | | `capabilities` | object | Optional expected results when a named capability is enabled; uses the same result format | +| `_skip` | array | Optional platforms that exclude this case | +| `_include` | array | Optional exclusive list of platforms that include this case | Example: @@ -215,6 +242,19 @@ Example: ] ``` +`_skip` and `_include` are mutually exclusive non-empty arrays of objects shaped +as `{"platform": "python", "reason": "optional explanation"}`. Each object +has exactly one required canonical `platform` and an optional non-empty +`reason`; a platform may appear only once. With neither field, the case applies +to every platform. `_skip` excludes listed platforms. `_include` includes only +listed platforms. The canonical platform list and fixture validation are in +[`ci/validate-fixtures.py`](ci/validate-fixtures.py). + +The root configuration and cases form the legacy `core` suite. New feature +suites may be self-contained under `evaluation-suites//`, with their own +`ufc-config.json` and `cases/` directory. Consumers explicitly opt into suites +and must not load the configuration for a suite they have not enabled. + ### Optional capabilities Every case still runs with default provider options and checks `result`. diff --git a/ci/test_validate_fixtures.py b/ci/test_validate_fixtures.py index a4f21c08..c5a9064f 100644 --- a/ci/test_validate_fixtures.py +++ b/ci/test_validate_fixtures.py @@ -1,7 +1,9 @@ -"""Regression checks for optional selected split/allocation expectations.""" +"""Regression checks for rules-based evaluation fixture validation.""" import importlib.util +import json from pathlib import Path +import tempfile import unittest @@ -90,5 +92,114 @@ def test_capability_feature_expectations_are_also_validated(self) -> None: ) +class EvaluationSuiteValidationTest(unittest.TestCase): + platforms = frozenset({"php", "python"}) + + @staticmethod + def config(flag: str) -> dict[str, object]: + return { + "createdAt": "2026-01-01T00:00:00Z", + "format": "SERVER", + "environment": {"name": "test"}, + "flags": { + flag: { + "key": flag, + "variations": {"on": {"key": "on", "value": True}}, + "allocations": [], + } + }, + } + + @staticmethod + def case(flag: str) -> dict[str, object]: + return { + "flag": flag, + "variationType": "BOOLEAN", + "defaultValue": False, + "targetingKey": "subject", + "attributes": {}, + "result": {"value": True, "reason": "STATIC"}, + } + + def test_legacy_and_directory_core_cannot_both_exist(self) -> None: + suite_path = validator.EVALUATION_SUITES_DIRECTORY / "core" + with self.assertRaisesRegex(validator.FixtureValidationError, "cannot exist in both"): + validator._validate_suite_id(suite_path, legacy_core_exists=True) + + def test_self_contained_suite_uses_its_adjacent_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + suite = Path(temporary_directory) / "example" + cases = suite / "cases" + cases.mkdir(parents=True) + config_path = suite / "ufc-config.json" + config_path.write_text(json.dumps(self.config("suite-flag")), encoding="utf-8") + (cases / "test-suite.json").write_text( + json.dumps([self.case("suite-flag")]), encoding="utf-8", + ) + + self.assertEqual( + (1, 1), + validator._validate_suite(config_path, cases, self.platforms, self_contained=True), + ) + + (cases / "test-suite.json").write_text( + json.dumps([self.case("root-only-flag")]), encoding="utf-8", + ) + with self.assertRaisesRegex(validator.FixtureValidationError, "unknown flag 'root-only-flag'"): + validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + + def test_self_contained_suite_rejects_missing_or_extra_entries(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + suite = Path(temporary_directory) / "example" + suite.mkdir() + config_path = suite / "ufc-config.json" + config_path.write_text(json.dumps(self.config("suite-flag")), encoding="utf-8") + cases = suite / "cases" + + with self.assertRaisesRegex(validator.FixtureValidationError, "must be a directory"): + validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + + cases.mkdir() + (cases / "test-suite.json").write_text( + json.dumps([self.case("suite-flag")]), encoding="utf-8", + ) + (suite / "unexpected.json").write_text("{}", encoding="utf-8") + with self.assertRaisesRegex(validator.FixtureValidationError, "unexpected suite entry"): + validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + + +class EvaluationPlatformValidationTest(unittest.TestCase): + platforms = frozenset({"php", "python"}) + path = validator.EVALUATION_CASES_DIRECTORY / "test-case-filter.json" + + def validate(self, value: dict[str, object]) -> None: + validator._validate_platform_filter(self.path, "case 0", value, self.platforms) + + def test_canonical_platforms_cover_annotated_sdks(self) -> None: + self.assertIn("php", validator.CANONICAL_PLATFORMS) + self.assertIn("python", validator.CANONICAL_PLATFORMS) + + def test_skip_and_include_accept_canonical_platforms(self) -> None: + self.validate({"_skip": [{"platform": "php", "reason": "Known gap."}]}) + self.validate({"_include": [{"platform": "python"}]}) + + def test_skip_and_include_are_mutually_exclusive(self) -> None: + with self.assertRaisesRegex(validator.FixtureValidationError, "mutually exclusive"): + self.validate({"_skip": [{"platform": "php"}], "_include": [{"platform": "python"}]}) + + def test_filters_reject_invalid_shapes_and_platforms(self) -> None: + invalid = ( + ({"_skip": []}, "non-empty array"), + ({"_skip": [{"platform": "unknown"}]}, "canonical platform"), + ({"_skip": [{"platform": "php"}, {"platform": "php"}]}, "repeats platform"), + ({"_skip": [{"platform": "php", "reason": ""}]}, "non-empty string"), + ({"_skip": [{"platform": "php", "ticket": "ABC-1"}]}, "unexpected fields"), + ) + for value, message in invalid: + with self.subTest(value=value): + with self.assertRaisesRegex(validator.FixtureValidationError, message): + self.validate(value) + + if __name__ == "__main__": unittest.main() diff --git a/ci/validate-fixtures.py b/ci/validate-fixtures.py index 24733e60..6c290e1f 100755 --- a/ci/validate-fixtures.py +++ b/ci/validate-fixtures.py @@ -5,6 +5,7 @@ import json import math +import re from pathlib import Path from typing import NoReturn @@ -12,6 +13,11 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] CONFIG_PATH = REPOSITORY_ROOT / "ufc-config.json" EVALUATION_CASES_DIRECTORY = REPOSITORY_ROOT / "evaluation-cases" +EVALUATION_SUITES_DIRECTORY = REPOSITORY_ROOT / "evaluation-suites" +SUITE_NAME_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +CANONICAL_PLATFORMS = frozenset( + {"cpp", "dotnet", "golang", "java", "nodejs", "php", "python", "ruby", "rust", "web"} +) VALID_VARIATION_TYPES = frozenset({"BOOLEAN", "INTEGER", "JSON", "NUMERIC", "STRING"}) FEATURE_DESTINATIONS = frozenset({"HOOK", "EXPOSURE", "EVALUATION"}) REQUIRED_CASE_FIELDS = frozenset( @@ -31,7 +37,10 @@ class FixtureValidationError(ValueError): def _relative(path: Path) -> Path: - return path.relative_to(REPOSITORY_ROOT) + try: + return path.relative_to(REPOSITORY_ROOT) + except ValueError: + return path def _fail(path: Path, message: str) -> NoReturn: @@ -56,62 +65,100 @@ def _load_json(path: Path) -> object: _fail(path, str(error)) -def _validate_config(value: object) -> dict[str, object]: +def _validate_config(path: Path, value: object) -> dict[str, object]: if not isinstance(value, dict): - _fail(CONFIG_PATH, "top-level value must be an object") + _fail(path, "top-level value must be an object") if not isinstance(value.get("createdAt"), str) or not value["createdAt"]: - _fail(CONFIG_PATH, "top-level createdAt must be a non-empty string") + _fail(path, "top-level createdAt must be a non-empty string") if not isinstance(value.get("format"), str) or not value["format"]: - _fail(CONFIG_PATH, "top-level format must be a non-empty string") + _fail(path, "top-level format must be a non-empty string") environment = value.get("environment") if not isinstance(environment, dict): - _fail(CONFIG_PATH, "top-level environment must be an object") + _fail(path, "top-level environment must be an object") if not isinstance(environment.get("name"), str) or not environment["name"]: - _fail(CONFIG_PATH, "environment.name must be a non-empty string") + _fail(path, "environment.name must be a non-empty string") flags = value.get("flags") if not isinstance(flags, dict): - _fail(CONFIG_PATH, "top-level flags must be an object") + _fail(path, "top-level flags must be an object") if not flags: - _fail(CONFIG_PATH, "top-level flags must not be empty") + _fail(path, "top-level flags must not be empty") for flag_name, flag_value in flags.items(): if not isinstance(flag_name, str) or not flag_name: - _fail(CONFIG_PATH, "every flag map key must be a non-empty string") + _fail(path, "every flag map key must be a non-empty string") if not isinstance(flag_value, dict): - _fail(CONFIG_PATH, f"flag {flag_name!r} must be an object") + _fail(path, f"flag {flag_name!r} must be an object") if flag_value.get("key") != flag_name: - _fail(CONFIG_PATH, f"flag {flag_name!r} must repeat its map key in the key field") + _fail(path, f"flag {flag_name!r} must repeat its map key in the key field") variations = flag_value.get("variations") if not isinstance(variations, dict): - _fail(CONFIG_PATH, f"flag {flag_name!r} variations must be an object") + _fail(path, f"flag {flag_name!r} variations must be an object") for variation_name, variation_value in variations.items(): if not isinstance(variation_value, dict): - _fail(CONFIG_PATH, f"flag {flag_name!r} variation {variation_name!r} must be an object") + _fail(path, f"flag {flag_name!r} variation {variation_name!r} must be an object") if variation_value.get("key") != variation_name: _fail( - CONFIG_PATH, + path, f"flag {flag_name!r} variation {variation_name!r} must repeat its map key in the key field", ) if "allocations" not in flag_value: - _fail(CONFIG_PATH, f"flag {flag_name!r} must declare allocations") + _fail(path, f"flag {flag_name!r} must declare allocations") return flags +def _validate_platform_filter( + path: Path, + location: str, + value: dict[str, object], + platforms: frozenset[str], +) -> None: + if "_skip" in value and "_include" in value: + _fail(path, f"{location} _skip and _include are mutually exclusive") + + for field in ("_skip", "_include"): + if field not in value: + continue + entries = value[field] + if not isinstance(entries, list) or not entries: + _fail(path, f"{location} {field} must be a non-empty array") + seen: set[str] = set() + for entry_index, entry in enumerate(entries): + entry_location = f"{location} {field}[{entry_index}]" + if not isinstance(entry, dict): + _fail(path, f"{entry_location} must be an object") + unexpected = sorted(set(entry).difference({"platform", "reason"})) + if unexpected: + _fail(path, f"{entry_location} has unexpected fields: {', '.join(unexpected)}") + platform = entry.get("platform") + if not isinstance(platform, str) or platform not in platforms: + _fail(path, f"{entry_location}.platform must be a canonical platform") + if platform in seen: + _fail(path, f"{location} {field} repeats platform {platform!r}") + seen.add(platform) + if "reason" in entry and (not isinstance(entry["reason"], str) or not entry["reason"]): + _fail(path, f"{entry_location}.reason must be a non-empty string") + + def _validate_evaluation_case( path: Path, index: int, value: object, configured_flags: dict[str, object], + platforms: frozenset[str] | None = None, ) -> None: location = f"case {index}" if not isinstance(value, dict): _fail(path, f"{location} must be an object") + if platforms is None: + platforms = CANONICAL_PLATFORMS + _validate_platform_filter(path, location, value, platforms) + missing_fields = sorted(REQUIRED_CASE_FIELDS.difference(value)) if missing_fields: _fail(path, f"{location} is missing required fields: {', '.join(missing_fields)}") @@ -194,12 +241,35 @@ def _validate_features(path: Path, location: str, features: object) -> None: _fail(path, f"{location}.{destination}.{key} must be a string, boolean, or finite number") -def main() -> None: - configured_flags = _validate_config(_load_json(CONFIG_PATH)) - - evaluation_paths = sorted(EVALUATION_CASES_DIRECTORY.glob("*.json")) +def _validate_suite( + config_path: Path, + cases_directory: Path, + platforms: frozenset[str], + *, + self_contained: bool = False, +) -> tuple[int, int]: + if config_path.is_symlink() or not config_path.is_file(): + _fail(config_path, "must be a regular file, not a symlink") + if cases_directory.is_symlink() or not cases_directory.is_dir(): + _fail(cases_directory, "must be a directory, not a symlink") + + if self_contained: + suite_directory = config_path.parent + allowed_entries = {config_path.name, cases_directory.name} + for entry in suite_directory.iterdir(): + if entry.name not in allowed_entries: + _fail(entry, "unexpected suite entry; only ufc-config.json and cases/ are allowed") + + configured_flags = _validate_config(config_path, _load_json(config_path)) + evaluation_paths: list[Path] = [] + for entry in sorted(cases_directory.iterdir()): + if entry.is_symlink() or not entry.is_file(): + _fail(entry, "case entries must be regular files, not directories or symlinks") + if entry.suffix != ".json": + _fail(entry, "case filenames must end in .json") + evaluation_paths.append(entry) if not evaluation_paths: - _fail(EVALUATION_CASES_DIRECTORY, "must contain at least one JSON file") + _fail(cases_directory, "must contain at least one JSON file") case_count = 0 for path in evaluation_paths: @@ -210,12 +280,53 @@ def main() -> None: _fail(path, "evaluation-case array must not be empty") for index, case in enumerate(cases): - _validate_evaluation_case(path, index, case, configured_flags) + _validate_evaluation_case(path, index, case, configured_flags, platforms) case_count += 1 + return case_count, len(evaluation_paths) + + +def _validate_suite_id(suite_directory: Path, *, legacy_core_exists: bool) -> None: + if not SUITE_NAME_PATTERN.fullmatch(suite_directory.name): + _fail(suite_directory, "suite directory name must be lowercase kebab-case") + if legacy_core_exists and suite_directory.name == "core": + _fail(suite_directory, "core cannot exist in both the legacy and evaluation-suites layouts") + + +def main() -> None: + total_cases, total_files = _validate_suite( + CONFIG_PATH, + EVALUATION_CASES_DIRECTORY, + CANONICAL_PLATFORMS, + ) + suite_count = 1 + + if EVALUATION_SUITES_DIRECTORY.exists(): + if EVALUATION_SUITES_DIRECTORY.is_symlink() or not EVALUATION_SUITES_DIRECTORY.is_dir(): + _fail(EVALUATION_SUITES_DIRECTORY, "must be a directory, not a symlink") + suite_directories: list[Path] = [] + for entry in sorted(EVALUATION_SUITES_DIRECTORY.iterdir()): + if entry.is_symlink() or not entry.is_dir(): + _fail(entry, "suite entries must be directories, not files or symlinks") + suite_directories.append(entry) + if not suite_directories: + _fail(EVALUATION_SUITES_DIRECTORY, "must contain at least one suite directory") + + for suite_directory in suite_directories: + _validate_suite_id(suite_directory, legacy_core_exists=True) + case_count, file_count = _validate_suite( + suite_directory / "ufc-config.json", + suite_directory / "cases", + CANONICAL_PLATFORMS, + self_contained=True, + ) + total_cases += case_count + total_files += file_count + suite_count += 1 + print( - f"Validated {_relative(CONFIG_PATH)} and {case_count} evaluation cases " - f"across {len(evaluation_paths)} files." + f"Validated {suite_count} evaluation suite(s), {total_cases} evaluation cases " + f"across {total_files} files." ) diff --git a/evaluation-cases/test-case-disabled-flag.json b/evaluation-cases/test-case-disabled-flag.json index 208e5c96..2278ff33 100644 --- a/evaluation-cases/test-case-disabled-flag.json +++ b/evaluation-cases/test-case-disabled-flag.json @@ -1,5 +1,11 @@ [ { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "country": "US", "email": "alice@mycompany.com" @@ -14,6 +20,12 @@ "variationType": "INTEGER" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "country": "Canada", "email": "bob@example.com" @@ -28,6 +40,12 @@ "variationType": "INTEGER" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for disabled flags." + } + ], "attributes": { "age": 50 }, diff --git a/evaluation-cases/test-case-empty-flag.json b/evaluation-cases/test-case-empty-flag.json index 18ee5381..f2805230 100644 --- a/evaluation-cases/test-case-empty-flag.json +++ b/evaluation-cases/test-case-empty-flag.json @@ -1,5 +1,11 @@ [ { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "country": "US", "email": "alice@mycompany.com" @@ -14,6 +20,12 @@ "variationType": "STRING" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "country": "Canada", "email": "bob@example.com" @@ -28,6 +40,12 @@ "variationType": "STRING" }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports PARSE_ERROR for empty flags." + } + ], "attributes": { "age": 50 }, diff --git a/evaluation-cases/test-case-experimentation-opt-out.json b/evaluation-cases/test-case-experimentation-opt-out.json index daa4f725..69ca8cca 100644 --- a/evaluation-cases/test-case-experimentation-opt-out.json +++ b/evaluation-cases/test-case-experimentation-opt-out.json @@ -9,6 +9,12 @@ } }, { + "_skip": [ + { + "platform": "php", + "reason": "PHP reports GENERAL when the targeting key is missing." + } + ], "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": null, "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, "result": { "value": {}, "reason": "ERROR", "errorCode": "TARGETING_KEY_MISSING" }, @@ -216,6 +222,12 @@ } }, { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects object opt-out attributes before flag evaluation." + } + ], "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", "attributes": { "dd_experimentation_opt_out_at": { "timestamp": "2026-10-19T12:00:00Z" } }, "result": { "value": 1, "reason": "SPLIT" }, @@ -225,6 +237,12 @@ } }, { + "_skip": [ + { + "platform": "python", + "reason": "Python rejects array opt-out attributes before flag evaluation." + } + ], "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", "attributes": { "dd_experimentation_opt_out_at": ["2026-10-19T12:00:00Z"] }, "result": { "value": 1, "reason": "SPLIT" },