From deec34b0a6b9dce3b78a3c12cf06fa7022433ce7 Mon Sep 17 00:00:00 2001 From: "tyler.potter" Date: Wed, 14 Oct 2026 16:04:06 +0000 Subject: [PATCH 1/6] test(ffe): add dependent flag evaluation coverage Add dependent-flag UFC fixtures and validate successful and failed root evaluation results through the FFE system-test harness. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + manifests/rust.yml | 1 + tests/parametric/test_ffe/flags-v1.json | 489 +++++++++++++++++- .../test_ffe/test-case-dependent-flags.json | 91 ++++ .../test_ffe/test_dynamic_evaluation.py | 65 ++- .../build/docker/nodejs/parametric/server.js | 24 +- 12 files changed, 636 insertions(+), 41 deletions(-) create mode 100644 tests/parametric/test_ffe/test-case-dependent-flags.json diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 7faed43c30f..2e7baad2115 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1169,6 +1169,7 @@ manifest: - declaration: bug (FFL-3325) component_version: <3.55.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script tests/parametric/test_ffe/test_span_enrichment.py: '>=3.54.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/golang.yml b/manifests/golang.yml index 2777e0c5173..349f2fc01dc 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2137,6 +2137,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_span_enrichment.py: '>=2.10.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index 22fdb616f37..01497377ee2 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4231,6 +4231,7 @@ manifest: : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_span_enrichment.py: '>=1.64.2' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index f7199ab57a2..fa82f4b92af 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2644,6 +2644,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: *ref_5_56_0 tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_span_enrichment.py: # TODO: a lower version might be supported - declaration: missing_feature (dd-trace-js#8343) component_version: <6.19.0 diff --git a/manifests/php.yml b/manifests/php.yml index d75e9410d24..6b5e48aedbd 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1585,6 +1585,7 @@ manifest: : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_span_enrichment.py: '>=1.23.3' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index 29b016149e4..21a0b3b9764 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2235,6 +2235,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_span_enrichment.py: '>=4.9.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Default_Fallback: missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index e71855fb615..fa6adfc94e7 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2662,6 +2662,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: missing_feature (FFL-2701 tracks Ruby agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_extract_invalid: # Easy win for all weblogs and version 2.27.0 - declaration: missing_feature (Missing for 2.x) diff --git a/manifests/rust.yml b/manifests/rust.yml index 088987f84e8..550f15fd78b 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -241,6 +241,7 @@ manifest: tests/parametric/test_extract_behavior.py: '>=0.4.1-dev' tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported + tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_detailed_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/tests/parametric/test_ffe/flags-v1.json b/tests/parametric/test_ffe/flags-v1.json index f01bfe2ef86..1b9f38a8467 100644 --- a/tests/parametric/test_ffe/flags-v1.json +++ b/tests/parametric/test_ffe/flags-v1.json @@ -4,6 +4,9 @@ "environment": { "name": "Test" }, + "evaluatorParams": { + "maxDependencyDepth": 2 + }, "flags": { "empty_flag": { "key": "empty_flag", @@ -3293,6 +3296,490 @@ "doLog": true } ] + }, + "dependent-leaf-on": { + "key": "dependent-leaf-on", + "enabled": true, + "variationType": "STRING", + "variations": { + "on": { + "key": "on", + "value": "leaf-on" + } + }, + "allocations": [ + { + "key": "leaf-static", + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-direct-root": { + "key": "dependent-direct-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "direct-match" + }, + "fallback": { + "key": "fallback", + "value": "direct-fallback" + } + }, + "allocations": [ + { + "key": "dependent-match", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-transitive-parent": { + "key": "dependent-transitive-parent", + "enabled": true, + "variationType": "STRING", + "variations": { + "parent-on": { + "key": "parent-on", + "value": "parent-on" + }, + "parent-off": { + "key": "parent-off", + "value": "parent-off" + } + }, + "allocations": [ + { + "key": "leaf-on", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "parent-on", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "parent-off", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-transitive-root": { + "key": "dependent-transitive-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "transitive-match" + }, + "fallback": { + "key": "fallback", + "value": "transitive-fallback" + } + }, + "allocations": [ + { + "key": "parent-match", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-transitive-parent" + }, + "operator": "ONE_OF", + "value": [ + "parent-on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-depth-three-parent": { + "key": "dependent-depth-three-parent", + "enabled": true, + "variationType": "STRING", + "variations": { + "ready": { + "key": "ready", + "value": "ready" + } + }, + "allocations": [ + { + "key": "transitive-parent-ready", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-transitive-parent" + }, + "operator": "ONE_OF", + "value": [ + "parent-on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "ready", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-depth-three-root": { + "key": "dependent-depth-three-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "over-depth", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-depth-three-parent" + }, + "operator": "ONE_OF", + "value": [ + "ready" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-short-circuit-root": { + "key": "dependent-short-circuit-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "short-circuit-fallback" + } + }, + "allocations": [ + { + "key": "short-circuited-rule", + "rules": [ + { + "conditions": [ + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + }, + { + "flagEvaluation": { + "key": "dependent-missing-prerequisite" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-memoized-root": { + "key": "dependent-memoized-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "memoized-match" + } + }, + "allocations": [ + { + "key": "same-prerequisite-twice", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "NOT_ONE_OF", + "value": [ + "off" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-atomic-error-root": { + "key": "dependent-atomic-error-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "present-then-missing", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "flagEvaluation": { + "key": "dependent-missing-prerequisite" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-cycle-a": { + "key": "dependent-cycle-a", + "enabled": true, + "variationType": "STRING", + "variations": { + "a": { + "key": "a", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "a-needs-b", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-cycle-b" + }, + "operator": "ONE_OF", + "value": [ + "b" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "a", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-cycle-b": { + "key": "dependent-cycle-b", + "enabled": true, + "variationType": "STRING", + "variations": { + "b": { + "key": "b", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "b-needs-a", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-cycle-a" + }, + "operator": "ONE_OF", + "value": [ + "a" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "b", + "shards": [] + } + ], + "doLog": true + } + ] } } -} \ No newline at end of file +} diff --git a/tests/parametric/test_ffe/test-case-dependent-flags.json b/tests/parametric/test_ffe/test-case-dependent-flags.json new file mode 100644 index 00000000000..42aab86c1f9 --- /dev/null +++ b/tests/parametric/test_ffe/test-case-dependent-flags.json @@ -0,0 +1,91 @@ +[ + { + "description": "A root flag matches a direct prerequisite by variant key.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-direct-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "direct-match" + }, + "targetingKey": "dependent-direct", + "variationType": "STRING" + }, + { + "description": "A depth-two tree evaluates its leaf and intermediate prerequisite before matching the root.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-transitive-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "transitive-match" + }, + "targetingKey": "dependent-transitive", + "variationType": "STRING" + }, + { + "description": "A false attribute condition short-circuits before an otherwise fatal missing prerequisite. Only the root fallback is evaluated and exposed.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-short-circuit-root", + "result": { + "reason": "STATIC", + "value": "short-circuit-fallback" + }, + "targetingKey": "dependent-short-circuit", + "variationType": "STRING" + }, + { + "description": "Two membership conditions against the same prerequisite reuse one per-root result. A single prerequisite exposure and evaluation event make memoization observable without test-only traversal instrumentation.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-memoized-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "memoized-match" + }, + "targetingKey": "dependent-memoized", + "variationType": "STRING" + }, + { + "description": "A successful prerequisite followed by a missing prerequisite aborts the whole graph. The root returns the caller default with the propagated missing-flag error. Exposure telemetry rolls back atomically, while the successful prerequisite and failed root retain evaluation telemetry.", + "attributes": {}, + "defaultValue": "atomic-default", + "flag": "dependent-atomic-error-root", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": "atomic-default" + }, + "targetingKey": "dependent-atomic-error", + "variationType": "STRING" + }, + { + "description": "With maxDependencyDepth 2, traversal from the root through two prerequisites is allowed, but the third dependency edge aborts the graph. Exposures roll back, while every affected ancestor emits its propagated error evaluation.", + "attributes": {}, + "defaultValue": "depth-default", + "flag": "dependent-depth-three-root", + "result": { + "errorCode": "GENERAL", + "reason": "ERROR", + "value": "depth-default" + }, + "targetingKey": "dependent-over-depth", + "variationType": "STRING" + }, + { + "description": "A dependency cycle aborts the whole graph and returns a normalized root error. Exposures roll back, while each affected flag emits one propagated error evaluation.", + "attributes": {}, + "defaultValue": "cycle-default", + "flag": "dependent-cycle-a", + "result": { + "errorCode": "GENERAL", + "reason": "ERROR", + "value": "cycle-default" + }, + "targetingKey": "dependent-cycle", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 8aa1b906aaf..f73404777a1 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -34,20 +34,29 @@ def _load_ufc_fixture() -> dict[str, Any]: return json.load(f) -def _get_test_case_files() -> list[str]: - """Get all test case files from the fixtures directory.""" +def _get_test_case_files() -> tuple[list[str], list[str]]: + """Get test case files grouped by whether they assert evaluation details.""" test_data_dir = Path(__file__).parent if not test_data_dir.exists(): - return [] + return [], [] # Exclude base fixtures that aren't test cases excluded = {"flags-v1.json", "span-enrichment-flags.json"} - return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] + value_only: list[str] = [] + detailed: list[str] = [] + for test_case_path in sorted(test_data_dir.glob("*.json")): + if test_case_path.name in excluded: + continue + with test_case_path.open() as f: + test_cases = json.load(f) + destination = detailed if any(set(test_case["result"]) != {"value"} for test_case in test_cases) else value_only + destination.append(test_case_path.name) + return value_only, detailed # Load fixture at module level for reuse across tests UFC_FIXTURE_DATA = _load_ufc_fixture() -ALL_TEST_CASE_FILES = _get_test_case_files() +VALUE_ONLY_TEST_CASE_FILES, DETAILED_TEST_CASE_FILES = _get_test_case_files() DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", @@ -146,18 +155,12 @@ def test_ffe_remote_config(self, test_agent: TestAgentAPI, test_library: APMLibr assert apply_state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value assert apply_state["product"] == RC_PRODUCT - @parametrize("library_env", [{**DEFAULT_ENVVARS}]) - @parametrize("test_case_file", ALL_TEST_CASE_FILES) - def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: - """Test FFE flag evaluation logic with various targeting scenarios. - - This is the core FFE test that validates the OpenFeature provider correctly: - 1. Loads flag configurations from Remote Config (UFC format) - 2. Evaluates flags based on targeting rules and evaluation context - 3. Returns correct variation values for different variation types - 4. Handles user targeting, attribute matching, and rollout percentages - - """ + def _assert_evaluation_cases( + self, + test_case_file: str, + test_agent: TestAgentAPI, + test_library: APMLibrary, + ) -> None: # Load the test case file test_case_path = Path(__file__).parent / test_case_file @@ -181,7 +184,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI default_value = test_case["defaultValue"] targeting_key = test_case["targetingKey"] attributes = test_case.get("attributes", {}) - expected_result = test_case["result"]["value"] + expected_result = test_case["result"] result = _ffe_evaluate_with_rc_retry( test_library, @@ -195,11 +198,23 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI f"Test case {i} in {test_case_file} failed: FFE provider did not load RC data after " f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" ) - actual_value = result.get("value") + for field, expected_value in expected_result.items(): + assert result.get(field) == expected_value, ( + f"Test case {i} in {test_case_file} failed: " + f"flag='{flag}', targetingKey='{targeting_key}', field='{field}', " + f"expected={expected_value!r}, actual={result.get(field)!r}" + ) - # Assert the evaluation result matches expected value - assert actual_value == expected_result, ( - f"Test case {i} in {test_case_file} failed: " - f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={actual_value}" - ) + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + @parametrize("test_case_file", VALUE_ONLY_TEST_CASE_FILES) + def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """Test FFE flag evaluation logic with various targeting scenarios.""" + self._assert_evaluation_cases(test_case_file, test_agent, test_library) + + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + @parametrize("test_case_file", DETAILED_TEST_CASE_FILES) + def test_ffe_detailed_flag_evaluation( + self, test_case_file: str, test_agent: TestAgentAPI, test_library: APMLibrary + ) -> None: + """Test fields requested by detailed evaluation cases.""" + self._assert_evaluation_cases(test_case_file, test_agent, test_library) diff --git a/utils/build/docker/nodejs/parametric/server.js b/utils/build/docker/nodejs/parametric/server.js index 933bdf836a1..dbc42d47d64 100644 --- a/utils/build/docker/nodejs/parametric/server.js +++ b/utils/build/docker/nodejs/parametric/server.js @@ -609,7 +609,7 @@ app.post('/ffe/start', async (req, res) => { app.post('/ffe/evaluate', async (req, res) => { const { flag, variationType, defaultValue, targetingKey, attributes, span_id } = req.body; - let value, reason; + let details; const context = { targetingKey, ...attributes } // Helper function to perform the actual flag evaluation @@ -617,28 +617,25 @@ app.post('/ffe/evaluate', async (req, res) => { try { switch (variationType) { case 'BOOLEAN': - value = await openFeatureClient.getBooleanValue(flag, defaultValue, context) + details = await openFeatureClient.getBooleanDetails(flag, defaultValue, context) break; case 'STRING': - value = await openFeatureClient.getStringValue(flag, defaultValue, context) + details = await openFeatureClient.getStringDetails(flag, defaultValue, context) break; case 'INTEGER': - value = await openFeatureClient.getNumberValue(flag, defaultValue, context) + details = await openFeatureClient.getNumberDetails(flag, defaultValue, context) break; case 'NUMERIC': - value = await openFeatureClient.getNumberValue(flag, defaultValue, context) + details = await openFeatureClient.getNumberDetails(flag, defaultValue, context) break; case 'JSON': - value = await openFeatureClient.getObjectValue(flag, defaultValue, context) + details = await openFeatureClient.getObjectDetails(flag, defaultValue, context) break; default: - value = defaultValue; + details = { value: defaultValue, reason: 'DEFAULT' }; } - - reason = 'DEFAULT'; } catch (error) { - value = defaultValue; - reason = 'ERROR'; + details = { value: defaultValue, reason: 'ERROR' }; } } @@ -650,10 +647,7 @@ app.post('/ffe/evaluate', async (req, res) => { await doEvaluate() } - res.json({ - value: value, - reason: reason - }); + res.json(details); }); // OpenTelemetry Metrics Endpoints From 314fcc3405a041d67af6252842eaa8f468360732 Mon Sep 17 00:00:00 2001 From: "tyler.potter" Date: Wed, 21 Oct 2026 14:50:23 +0000 Subject: [PATCH 2/6] Restore legacy FFE fixture discovery Preserve exclusion-based JSON discovery while partitioning detailed evaluation cases. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- tests/parametric/test_ffe/test_dynamic_evaluation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 4b2a19be33a..48f8fa64986 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -40,9 +40,12 @@ def _get_test_case_files() -> tuple[list[str], list[str]]: if not test_data_dir.exists(): return [], [] + excluded = {"flags-v1.json", "serial-id-metadata-flags.json", "span-enrichment-flags.json"} value_only: list[str] = [] detailed: list[str] = [] - for test_case_path in sorted(test_data_dir.glob("test-case-*.json")): + for test_case_path in sorted( + path for path in test_data_dir.iterdir() if path.suffix == ".json" and path.name not in excluded + ): with test_case_path.open() as f: test_cases = json.load(f) destination = detailed if any(set(test_case["result"]) != {"value"} for test_case in test_cases) else value_only From e40fc66e9cb6764ba008583937d682cc024c6c85 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 21 Oct 2026 16:56:20 +0000 Subject: [PATCH 3/6] test(ffe): consume canonical fixture snapshot Replace local fixture copies with the shared corpus, preserve extracted metadata and span inputs, and add a reviewed snapshot update workflow. Environment: Datadog workspace --- .github/CODEOWNERS | 4 + ...elf.update-ffe-fixtures.create-pr.sts.yaml | 14 + .github/workflows/update-ffe-fixtures.yml | 87 + tests/ffe/README.md | 31 +- tests/ffe/test_flag_eval_evp.py | 2 +- .../test_ffe/ffe-system-test-data/SOURCE.md | 12 + .../test-case-boolean-false-assignment.json | 35 +- .../test-case-boolean-one-of-matches.json | 176 +- .../test-case-comparator-operator-flag.json | 63 +- .../test-case-disabled-flag.json | 41 +- .../test-case-empty-flag.json | 41 +- .../test-case-empty-string-variation.json | 35 +- .../test-case-experimentation-opt-out.json | 210 ++ .../test-case-falsy-value-assignments.json | 35 +- .../test-case-flag-with-empty-string.json | 22 +- .../test-case-integer-flag.json | 265 ++- .../test-case-invalid-condition-operands.json | 59 + .../test-case-invalid-regex-isolation.json | 17 + ...t-case-invalid-shard-bounds-isolation.json | 40 + .../test-case-kill-switch-flag.json | 292 +-- .../test-case-malformed-flag-isolation.json | 27 + .../test-case-microsecond-date-flag.json | 39 + ...t-case-missing-split-shards-isolation.json | 27 + .../test-case-new-user-onboarding-flag.json | 316 +-- .../test-case-no-allocations-flag.json | 49 +- .../test-case-null-operator-flag.json | 63 +- .../test-case-null-shard-range-isolation.json | 15 + .../test-case-null-targeting-key.json | 40 + .../test-case-numeric-flag.json | 41 +- .../test-case-numeric-one-of-default.json | 13 + .../test-case-numeric-one-of.json | 77 +- .../test-case-of-7-empty-targeting-key.json | 11 +- .../test-case-regex-flag.json | 122 + .../test-case-semver-comparison-flag.json | 169 ++ ...e-semver-invalid-comparand-categories.json | 120 + ...se-semver-invalid-comparand-waterfall.json | 17 + .../test-case-semver-precedence-flag.json | 170 ++ .../test-case-semver-validation-flag.json | 255 ++ .../test-case-semver-version-parts.json | 118 + .../test-case-serial-id-metadata.json | 97 + .../test-case-span-enrichment-inputs.json | 67 + .../test-case-start-and-end-date-flag.json | 41 +- .../test-case-unknown-fields-tolerance.json | 16 + .../test-case-unknown-operator-isolation.json | 31 + .../test-case-variant-type-mismatch.json | 27 + .../test-flag-that-does-not-exist.json | 46 + .../test-json-config-flag.json | 76 +- .../test-no-allocations-flag.json | 49 +- .../test-special-characters.json | 44 +- .../test-string-with-special-characters.json | 732 +++--- .../ufc-config.json} | 2113 ++++++++++++++++- .../test_ffe/serial-id-metadata-flags.json | 97 - .../test_ffe/span-enrichment-flags.json | 146 -- .../test_ffe/test-case-regex-flag.json | 53 - .../test-flag-that-does-not-exist.json | 40 - .../test_ffe/test_configuration_sources.py | 2 +- .../test_ffe/test_dynamic_evaluation.py | 44 +- .../test_ffe/test_serial_id_metadata.py | 15 +- .../test_ffe/test_span_enrichment.py | 21 +- tests/parametric/test_ffe/utils.py | 14 + .../test_the_test/test_update_ffe_fixtures.py | 145 ++ .../_test_clients/_test_client_parametric.py | 4 +- utils/mocked_backend/ffe.py | 2 +- utils/scripts/update_ffe_fixtures.py | 206 ++ 64 files changed, 5761 insertions(+), 1537 deletions(-) create mode 100644 .github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml create mode 100644 .github/workflows/update-ffe-fixtures.yml create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-boolean-false-assignment.json (74%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-boolean-one-of-matches.json (72%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-comparator-operator-flag.json (68%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-disabled-flag.json (59%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-empty-flag.json (63%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-empty-string-variation.json (75%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-falsy-value-assignments.json (73%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-flag-with-empty-string.json (72%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-integer-flag.json (66%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-kill-switch-flag.json (67%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-new-user-onboarding-flag.json (69%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-no-allocations-flag.json (68%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-null-operator-flag.json (67%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-numeric-flag.json (52%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-numeric-one-of.json (70%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-of-7-empty-targeting-key.json (73%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-case-start-and-end-date-flag.json (65%) create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-json-config-flag.json (62%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-no-allocations-flag.json (70%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-special-characters.json (77%) rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-cases}/test-string-with-special-characters.json (77%) rename tests/parametric/test_ffe/{flags-v1.json => ffe-system-test-data/ufc-config.json} (60%) delete mode 100644 tests/parametric/test_ffe/serial-id-metadata-flags.json delete mode 100644 tests/parametric/test_ffe/span-enrichment-flags.json delete mode 100644 tests/parametric/test_ffe/test-case-regex-flag.json delete mode 100644 tests/parametric/test_ffe/test-flag-that-does-not-exist.json create mode 100644 tests/test_the_test/test_update_ffe_fixtures.py create mode 100644 utils/scripts/update_ffe_fixtures.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 277408f8d82..641b874b47c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,9 @@ /tests/ @DataDog/system-tests-reviewers # FFE +/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/.github/workflows/update-ffe-fixtures.yml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/utils/scripts/update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/docker_fixtures/_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/parametric/Endpoints/FfeTestApi.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core @@ -23,6 +26,7 @@ /utils/build/docker/php/common/ffe.php @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /tests/ffe/ @DataDog/feature-flagging-and-experimentation-sdk /tests/test_the_test/test_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk +/tests/test_the_test/test_update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk /tests/parametric/test_ffe/ @DataDog/feature-flagging-and-experimentation-sdk # SDK cap diff --git a/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml new file mode 100644 index 00000000000..7aeba37ded2 --- /dev/null +++ b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @@ -0,0 +1,14 @@ +--- +issuer: https://token.actions.githubusercontent.com + +subject: repo:DataDog/system-tests:ref:refs/heads/main + +claim_pattern: + event_name: (workflow_dispatch|schedule) + ref: refs/heads/main + ref_protected: "true" + job_workflow_ref: DataDog/system-tests/\.github/workflows/update-ffe-fixtures\.yml@refs/heads/main + +permissions: + contents: write + pull_requests: write diff --git a/.github/workflows/update-ffe-fixtures.yml b/.github/workflows/update-ffe-fixtures.yml new file mode 100644 index 00000000000..778db406a4b --- /dev/null +++ b/.github/workflows/update-ffe-fixtures.yml @@ -0,0 +1,87 @@ +--- +name: Update FFE fixtures + +on: + workflow_dispatch: + inputs: + fixture_ref: + description: Branch, tag, or commit from DataDog/ffe-system-test-data + required: true + default: main + type: string + schedule: + - cron: "0 0 * * 0" + +concurrency: + group: update-ffe-fixtures + cancel-in-progress: false + +jobs: + update-ffe-fixtures: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + + - name: Update fixture snapshot + id: fixtures + env: + FFE_FIXTURE_REF: ${{ inputs.fixture_ref || 'main' }} + run: python3 utils/scripts/update_ffe_fixtures.py --ref "$FFE_FIXTURE_REF" + + - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 + if: steps.fixtures.outputs.changed == 'true' + id: octo-sts + with: + scope: DataDog/system-tests + policy: self.update-ffe-fixtures.create-pr + + - name: Check for an open fixture update PR + if: steps.fixtures.outputs.changed == 'true' + id: existing-pr + env: + GH_TOKEN: ${{ steps.octo-sts.outputs.token }} + run: | + count=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --head actions/update-ffe-fixtures --state open \ + --json headRefName,headRepositoryOwner,isCrossRepository \ + --jq '[.[] | select( + .headRefName == "actions/update-ffe-fixtures" and + .headRepositoryOwner.login == "DataDog" and + (.isCrossRepository | not) + )] | length') + echo "count=$count" >> "$GITHUB_OUTPUT" + + - name: Create Pull Request + if: >- + steps.fixtures.outputs.changed == 'true' && steps.existing-pr.outputs.count == '0' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.octo-sts.outputs.token }} + branch: actions/update-ffe-fixtures + base: main + commit-message: "test(ffe): update canonical fixtures" + title: "test(ffe): update canonical fixtures" + draft: true + sign-commits: true + add-paths: tests/parametric/test_ffe/ffe-system-test-data + body: | + ## Motivation + + Keep SDK behavior consistent with the shared FFE evaluator contract. + + ## Changes + + Refresh the checked-in snapshot to [${{ steps.fixtures.outputs.source_commit }}](https://github.com/DataDog/ffe-system-test-data/commit/${{ steps.fixtures.outputs.source_commit }}), containing ${{ steps.fixtures.outputs.fixture_count }} evaluation cases. + + Ordinary test runs use the snapshot without network access. Resolve newly exposed evaluator failures before merging; correct shared expectations upstream only when they are wrong. Scheduled updates leave an existing update PR untouched to preserve review and fixes. + + ## Workflow + + 1. Keep this PR as a draft while validating the updated fixtures against the supported SDKs. + 2. Follow the repository's [review workflow](https://github.com/DataDog/system-tests/blob/main/.github/pull_request_template.md). diff --git a/tests/ffe/README.md b/tests/ffe/README.md index be43490db3c..f1205b765d5 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -12,6 +12,35 @@ This directory contains system tests for the Feature Flags & Experimentation (FF | `test_flag_eval_metrics.py` | Evaluation metrics (OTel counter) | | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds | +## Canonical parametric fixtures + +The parametric evaluation suite reads the checked-in snapshot under +`tests/parametric/test_ffe/ffe-system-test-data`. Its upstream source and exact +commit are recorded in `SOURCE.md` inside that directory. + +Change shared evaluator cases in +[`DataDog/ffe-system-test-data`](https://github.com/DataDog/ffe-system-test-data) +first, then refresh the snapshot: + +```bash +python3 utils/scripts/update_ffe_fixtures.py --ref +``` + +The updater imports `ufc-config.json` and `evaluation-cases/*.json` only. Ordinary +test runs use those checked-in files without submodules or network access. A +weekly/manual workflow opens a signed draft PR when their contents change and +leaves any existing fixture update PR untouched. Review and resolve newly exposed +SDK failures before merging an update. + +Dynamic evaluation checks each case's value, reason, and explicit error code. +Cases with optional `capabilities` still run their baseline `result`; enabling +capability-specific provider options requires separate harness support. +Remote Configuration tests and the mocked agentless backend read the same UFC. +Serial-ID metadata and span-enrichment tests select their flag definitions from +the shared corpus using their dedicated evaluation-case files. Their integration +harnesses retain trace construction and metadata assertions; no separate local +configuration copies are needed. + ## Running FFE Tests ```bash @@ -197,7 +226,7 @@ The tests use `interfaces.agent.get_metrics()` to retrieve metrics from the agen ### 1. Test returns `parse_error` instead of expected reason - Check UFC fixture format (especially `totalShards` placement) -- Verify the fixture matches `flags-v1.json` format in dd-trace-py +- Verify the fixture matches the canonical `ffe-system-test-data/ufc-config.json` format ### 2. Test returns `static` instead of `split` - Need multiple variations with different shard ranges diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index bcf93e46a17..5e53cb1b8a8 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -242,7 +242,7 @@ def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None: class FlagevaluationEgressContract: """One flag-evaluation contract inherited by each supported topology adapter.""" - # Agentless scenarios preload flags-v1.json before the weblog starts. Use the + # Agentless scenarios preload the canonical UFC before the weblog starts. Use the # same known fixture key in every topology; the Agent scenario installs an # equivalent fixture through Remote Config below. flag_key = "empty-targeting-key-flag" diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md new file mode 100644 index 00000000000..7a06e3fc2be --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -0,0 +1,12 @@ +# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: e5a64a8be81e0f435dbb5c950c06d45d4969b505 + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. diff --git a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json similarity index 74% rename from tests/parametric/test_ffe/test-case-boolean-false-assignment.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json index cdd2b0a513d..ab718f5bee1 100644 --- a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json @@ -1,38 +1,41 @@ [ { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "alice", "attributes": { "should_disable_feature": true }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": false - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "bob", "attributes": { "should_disable_feature": false }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "charlie", "attributes": { "unknown_attribute": "value" }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "DEFAULT", "value": true - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json similarity index 72% rename from tests/parametric/test_ffe/test-case-boolean-one-of-matches.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json index f616614d936..392ec97ec0f 100644 --- a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json @@ -1,192 +1,208 @@ [ { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "one_of_flag": "True" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "not_matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "not_matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "haley", "attributes": { "not_matches_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "haley", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "ivy", "attributes": { "not_one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "ivy", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "julia", "attributes": { "not_one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "julia", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "kim", "attributes": { "not_one_of_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "kim", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "lucas", "attributes": { "not_one_of_flag": "true" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "lucas", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "mike", "attributes": { "not_one_of_flag": "false" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "mike", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "nicole", "attributes": { "null_flag": "null" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 5 - } + }, + "targetingKey": "nicole", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "owen", "attributes": { "null_flag": null }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "owen", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "pete", "attributes": {}, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "pete", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-comparator-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json index 2d94f30eb30..e4daa2de7c2 100644 --- a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "small" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "medium" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "size": 25 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "large" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-disabled-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json similarity index 59% rename from tests/parametric/test_ffe/test-case-disabled-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json index 0da79189ade..208e5c96943 100644 --- a/tests/parametric/test_ffe/test-case-disabled-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json similarity index 63% rename from tests/parametric/test_ffe/test-case-empty-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json index 52100b1fe47..18ee5381fdc 100644 --- a/tests/parametric/test_ffe/test-case-empty-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-string-variation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json similarity index 75% rename from tests/parametric/test_ffe/test-case-empty-string-variation.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json index 1af431f3db4..f1fa3994af2 100644 --- a/tests/parametric/test_ffe/test-case-empty-string-variation.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json @@ -1,38 +1,41 @@ [ { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "empty_user", "attributes": { "content_type": "minimal" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "empty_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "full_user", "attributes": { "content_type": "full" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "detailed_content" - } + }, + "targetingKey": "full_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "default_user", "attributes": { "content_type": "unknown" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "default_user", + "variationType": "STRING" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json new file mode 100644 index 00000000000..8120c5fb73d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json @@ -0,0 +1,210 @@ +[ + { + "flag": "unknown-opt-out-flag", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "default", "reason": "ERROR", "errorCode": "FLAG_NOT_FOUND" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": null, + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": {}, "reason": "ERROR", "errorCode": "TARGETING_KEY_MISSING" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": {}, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": {}, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 3.1415926, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": true, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123456789Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T08:00:00-04:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": true }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": null }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": 1792411200 }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": false }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": null }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": { "timestamp": "2026-10-19T12:00:00Z" } }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": ["2026-10-19T12:00:00Z"] }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + } +] diff --git a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json similarity index 73% rename from tests/parametric/test_ffe/test-case-falsy-value-assignments.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json index 4ad259620d8..6a8cee51bcc 100644 --- a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json @@ -1,38 +1,41 @@ [ { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "zero_user", "attributes": { "plan_tier": "free" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 0 - } + }, + "targetingKey": "zero_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "premium_user", "attributes": { "plan_tier": "premium" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 100 - } + }, + "targetingKey": "premium_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "unknown_user", "attributes": { "plan_tier": "trial" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "DEFAULT", "value": 999 - } + }, + "targetingKey": "unknown_user", + "variationType": "INTEGER" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json similarity index 72% rename from tests/parametric/test_ffe/test-case-flag-with-empty-string.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json index 0bf562efa35..6a5d9cd62fd 100644 --- a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json @@ -1,24 +1,26 @@ [ { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": {}, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "SPLIT", "value": "non_empty" - } + }, + "targetingKey": "bob", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-integer-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json similarity index 66% rename from tests/parametric/test_ffe/test-case-integer-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json index 5e930885b44..8ff11aec1fc 100644 --- a/tests/parametric/test_ffe/test-case-integer-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json @@ -1,244 +1,267 @@ [ { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "debra", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "1", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "1", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "2", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "2", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "3", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "3", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "4", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "4", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "5", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "5", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "6", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "6", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "7", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "7", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "8", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "8", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "9", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "9", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "10", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "10", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "11", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "11", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "12", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "12", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "13", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "13", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "14", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "14", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "15", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "15", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "16", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "16", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "17", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "17", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "18", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "18", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "19", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "19", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json new file mode 100644 index 00000000000..be226381455 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json @@ -0,0 +1,59 @@ +[ + { + "description": "GT requires a numeric configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "age": 42 + }, + "defaultValue": "default", + "flag": "invalid-gt-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-gt", + "variationType": "STRING" + }, + { + "description": "ONE_OF requires an array configured operand. A flag with a scalar operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "role": "admin" + }, + "defaultValue": "default", + "flag": "invalid-one-of-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-one-of", + "variationType": "STRING" + }, + { + "description": "IS_NULL requires a boolean configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "deleted_at": null + }, + "defaultValue": "default", + "flag": "invalid-is-null-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-is-null", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same configuration must still evaluate after malformed condition operands are isolated.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json new file mode 100644 index 00000000000..3ad596cbf06 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json @@ -0,0 +1,17 @@ +[ + { + "description": "The malformed flag uses the invalid regex *@datadoghq.com before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": { + "email": "harish@datadoghq.com" + }, + "defaultValue": "default", + "flag": "invalid-regex-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "harish", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json new file mode 100644 index 00000000000..ca8553f099c --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json @@ -0,0 +1,40 @@ +[ + { + "description": "The malformed flag has totalShards above the unsigned 32-bit range. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "overflow-total-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "The malformed flag has a negative shard range start. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "negative-shard-range-start-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after invalid shard-bound flags are skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-kill-switch-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-kill-switch-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json index 8f34a1bc3af..3af700dc711 100644 --- a/tests/parametric/test_ffe/test-case-kill-switch-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json @@ -1,290 +1,314 @@ [ { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "barbara", "attributes": { - "email": "barbara@example.com", - "country": "canada" + "country": "canada", + "email": "barbara@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "barbara", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "charlie", "attributes": { "age": 40 }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "debra", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "1", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "1", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "2", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 50 + "age": 50, + "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "3", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "4", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "5", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "6", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "7", "attributes": { - "country": "US", - "age": 12 + "age": 12, + "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "7", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "8", "attributes": { - "country": "Italy", - "age": 60 + "age": 60, + "country": "Italy" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "8", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "9", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "10", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "10", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "11", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "11", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "12", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "13", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "14", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "14", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "15", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "16", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "17", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "18", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "19", + "variationType": "BOOLEAN" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json new file mode 100644 index 00000000000..41672357c23 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has allocations encoded as a string instead of an array. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "malformed-allocations-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the malformed flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json new file mode 100644 index 00000000000..3dba337ad02 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json @@ -0,0 +1,39 @@ +[ + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "country": "US" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "1.0.0" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "charlie", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json new file mode 100644 index 00000000000..d6dfeb180ea --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has a split that omits required shards. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "missing-split-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the missing-shards flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json similarity index 69% rename from tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json index 8ed3e2a024f..bb08523de4e 100644 --- a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json @@ -1,318 +1,344 @@ [ { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "green" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "debra", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "purple" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "id": "override-id", - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com", + "id": "override-id" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "Zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "Zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "1", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "1", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "2", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 33 + "age": 33, + "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "3", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "4", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "5", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "6", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "7", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "7", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "8", "attributes": { "country": "Italy" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "8", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "9", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "10", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "10", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "11", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "11", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "12", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "13", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "14", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "14", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "15", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "16", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "17", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "18", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "19", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json index 132c39db32a..7d985b8efd4 100644 --- a/tests/parametric/test_ffe/test-case-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "hello": "world" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "hello": "world" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-case-null-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-null-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json index 09e5d78dacd..8063ff8fee3 100644 --- a/tests/parametric/test_ffe/test-case-null-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "charlie", "attributes": { "size": null }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json new file mode 100644 index 00000000000..15f4af5a77d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json @@ -0,0 +1,15 @@ +[ + { + "description": "The malformed flag has a null shard range before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "null-shard-range-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json new file mode 100644 index 00000000000..a2226d1b21e --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json @@ -0,0 +1,40 @@ +[ + { + "attributes": {}, + "defaultValue": 42, + "flag": "numeric_flag", + "result": { + "reason": "STATIC", + "value": 3.1415926 + }, + "targetingKey": null, + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "France" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "ERROR", + "value": "default" + }, + "targetingKey": null, + "variationType": "STRING" + }, + { + "attributes": { + "country": "France", + "email": "null@mycompany.com" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "TARGETING_MATCH", + "value": "green" + }, + "targetingKey": null, + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json similarity index 52% rename from tests/parametric/test_ffe/test-case-numeric-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json index 757f0f70e55..cc52dcdff4d 100644 --- a/tests/parametric/test_ffe/test-case-numeric-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "alice", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "bob", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json new file mode 100644 index 00000000000..d8a7e8ba092 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json @@ -0,0 +1,13 @@ +[ + { + "attributes": {}, + "defaultValue": 0, + "flag": "numeric-one-of", + "result": { + "reason": "DEFAULT", + "value": 0 + }, + "targetingKey": "race-default", + "variationType": "INTEGER" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-one-of.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json similarity index 70% rename from tests/parametric/test_ffe/test-case-numeric-one-of.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json index 9eaccbc477c..bfa957335d3 100644 --- a/tests/parametric/test_ffe/test-case-numeric-one-of.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json @@ -1,86 +1,93 @@ [ { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "number": 2 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "number": 3 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "number": 4 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "number": "1" }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "number": 123456789 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json similarity index 73% rename from tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json index 03f03ab1ecb..e99a207b6da 100644 --- a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json @@ -1,12 +1,13 @@ [ { - "flag": "empty-targeting-key-flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "", "attributes": {}, + "defaultValue": "default", + "flag": "empty-targeting-key-flag", "result": { + "reason": "STATIC", "value": "on-value" - } + }, + "targetingKey": "", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json new file mode 100644 index 00000000000..b01f6f1e5ca --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json @@ -0,0 +1,122 @@ +[ + { + "attributes": { + "comma_regex_case": "xxaaay" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "comma-quantifier" + }, + "targetingKey": "comma-quantifier", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "prefix exact suffix" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "literal-whitespace" + }, + "targetingKey": "literal-whitespace", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "exact" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "trimmed-whitespace-does-not-match", + "variationType": "STRING" + }, + { + "attributes": { + "email": "user.name+tag@capture.example" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "capturing-groups" + }, + "targetingKey": "capture", + "variationType": "STRING" + }, + { + "attributes": { + "email": "llega mañana temprano" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "unicode-word-boundary" + }, + "targetingKey": "unicode", + "variationType": "STRING" + }, + { + "attributes": { + "email": "alice@example.com", + "version": "1.15.0" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "partial-example" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "email": "bob@test.com", + "version": "0.20.1" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "test" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "charlie", + "variationType": "STRING" + }, + { + "attributes": { + "email": "derek@gmail.com", + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "derek", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json new file mode 100644 index 00000000000..bc6fe8b92bc --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json @@ -0,0 +1,169 @@ +[ + { + "attributes": { + "app_version": "1.2.3-alpha.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "1.2.3-alpha.5" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "carol", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.9.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "dave", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "eve", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "frank", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.0" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "grace", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "henry", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "mia", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "future" + }, + "targetingKey": "iris", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "5.6.7" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "jane", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "not-a-semver" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "kyle", + "variationType": "STRING" + }, + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "liam", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json new file mode 100644 index 00000000000..4f00961d9f9 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json @@ -0,0 +1,120 @@ +[ + { + "description": "A non-parseable SemVer comparand (\"not-a-version\") rejects the flag at parse time with a PARSE_ERROR rather than matching or coercing.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-syntax-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-syntax", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with repeated hyphens inside its prerelease identifiers (\"1.2.3----R-S.12.9.1--.12+meta\") is valid and matches normally.", + "attributes": { + "app_version": "1.2.3----R-S.12.9.1--.12+meta" + }, + "defaultValue": "unknown", + "flag": "semver-valid-comparand-hyphenated-prerelease-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "matched" + }, + "targetingKey": "valid-hyphenated-prerelease", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with consecutive dots in its prerelease identifiers (\"1.0.0-alpha..1\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-consecutive-dots-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-consecutive-dots", + "variationType": "STRING" + }, + { + "description": "A Ruby-style dot-suffixed SemVer comparand (\"1.2.3.DEV\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-dot-suffix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-dot-suffix", + "variationType": "STRING" + }, + { + "description": "A v-prefixed SemVer comparand (\"v1.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-vprefix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-vprefix", + "variationType": "STRING" + }, + { + "description": "A leading-zero core component in the SemVer comparand (\"01.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-leading-zero-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-leading-zero", + "variationType": "STRING" + }, + { + "description": "A non-string (JSON number) SemVer comparand rejects the flag at parse time with a PARSE_ERROR rather than coercing the number to a string.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-non-string-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-non-string", + "variationType": "STRING" + }, + { + "description": "The rejected invalid-comparand flags must not poison the rest of the configuration: a neighboring valid SemVer flag still evaluates normally.", + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "after-invalid-categories", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json new file mode 100644 index 00000000000..cbddbb87d03 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json @@ -0,0 +1,17 @@ +[ + { + "description": "An invalid configured SemVer comparand in a non-last allocation must reject the whole flag at parse time, even though a valid default allocation follows it in the waterfall. An implementation that skips the invalid allocation and falls through to the default would return \"fallback\" with a STATIC reason instead of an error.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-waterfall-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "waterfall", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json new file mode 100644 index 00000000000..ef027a06660 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json @@ -0,0 +1,170 @@ +[ + { + "description": "Multi-digit numeric prerelease identifiers compare by numeric value, not lexicographically: 1.0.0-beta.2 must precede 1.0.0-beta.11. An implementation that string-compares identifiers would treat \"2\" as greater than \"11\" and fail to match.", + "attributes": { + "app_version": "10.0.0-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "numeric-prerelease" + }, + "targetingKey": "numeric-prerelease", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for equality: 1.0.0+linux and 1.0.0+darwin have equal precedence, so SEMVER_EQ matches.", + "attributes": { + "app_version": "20.0.0+linux" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-equal-precedence" + }, + "targetingKey": "build-equal-precedence", + "variationType": "STRING" + }, + { + "description": "Dotted/multi-segment build metadata is ignored for equality: 4.0.0+exp.sha.5114f85 equals 4.0.0.", + "attributes": { + "app_version": "30.0.0+exp.sha.5114f85" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-dotted-eq" + }, + "targetingKey": "build-dotted-eq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LTE: 4.0.0+build.42 <= 4.0.0 is true (match).", + "attributes": { + "app_version": "40.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-lte" + }, + "targetingKey": "build-lte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GTE: 4.0.0+build.42 >= 4.0.0 is true (match).", + "attributes": { + "app_version": "50.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-gte" + }, + "targetingKey": "build-gte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_NEQ: 4.0.0+build.42 != 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "60.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-neq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LT: 4.0.0+build.42 < 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "70.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-lt", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GT: 4.0.0+build.42 > 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "80.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-gt", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: 1.0.0-alpha precedes 1.0.0-alpha.1 (fewer identifiers first), so SEMVER_LT matches.", + "attributes": { + "app_version": "90.0.0-alpha" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-before-alpha-1" + }, + "targetingKey": "alpha-before-alpha-1", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: numeric identifiers have lower precedence than alphanumeric (1.0.0-alpha.1 precedes 1.0.0-alpha.beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "91.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-1-before-alpha-beta" + }, + "targetingKey": "alpha-1-before-alpha-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: identifiers are compared lexicographically (1.0.0-alpha.beta precedes 1.0.0-beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "92.0.0-alpha.beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-beta-before-beta" + }, + "targetingKey": "alpha-beta-before-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: a prerelease version precedes the corresponding release (1.0.0-beta precedes 1.0.0), so SEMVER_LT matches.", + "attributes": { + "app_version": "93.0.0-beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta-before-release" + }, + "targetingKey": "beta-before-release", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json new file mode 100644 index 00000000000..9b4bfea3710 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json @@ -0,0 +1,255 @@ +[ + { + "description": "The maximum safe integer value is valid in every core version component.", + "attributes": { + "app_version": "9007199254740991.9007199254740991.9007199254740991" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "maximum" + }, + "targetingKey": "max-core", + "variationType": "STRING" + }, + { + "description": "Semver does not accept a v prefix.", + "attributes": { + "app_version": "v1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "v-prefix", + "variationType": "STRING" + }, + { + "description": "Core numeric components cannot have leading zeros.", + "attributes": { + "app_version": "01.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "core-leading-zero", + "variationType": "STRING" + }, + { + "description": "Numeric prerelease identifiers cannot have leading zeros.", + "attributes": { + "app_version": "1.2.3-01" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "prerelease-leading-zero", + "variationType": "STRING" + }, + { + "description": "Prerelease identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3-alpha..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease-identifier", + "variationType": "STRING" + }, + { + "description": "Build metadata identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3+build..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build-identifier", + "variationType": "STRING" + }, + { + "description": "Prerelease and build identifiers are ASCII-only.", + "attributes": { + "app_version": "1.2.3-α" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-ascii-prerelease", + "variationType": "STRING" + }, + { + "description": "An empty string is not a valid SemVer version.", + "attributes": { + "app_version": "" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-string", + "variationType": "STRING" + }, + { + "description": "A leading zero in the minor core component is rejected.", + "attributes": { + "app_version": "1.02.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "minor-leading-zero", + "variationType": "STRING" + }, + { + "description": "A leading zero in the patch core component is rejected.", + "attributes": { + "app_version": "1.2.03" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "patch-leading-zero", + "variationType": "STRING" + }, + { + "description": "A trailing prerelease delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3-" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease", + "variationType": "STRING" + }, + { + "description": "A trailing build delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3+" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build", + "variationType": "STRING" + }, + { + "description": "Underscores are not permitted in prerelease identifiers.", + "attributes": { + "app_version": "1.2.3-alpha_1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "underscore-prerelease", + "variationType": "STRING" + }, + { + "description": "More than one build metadata delimiter is rejected.", + "attributes": { + "app_version": "1.2.3-alpha+build+other" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "multiple-build-delimiters", + "variationType": "STRING" + }, + { + "description": "Leading whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": " 1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "leading-whitespace", + "variationType": "STRING" + }, + { + "description": "Trailing whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": "1.2.3 " + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "trailing-whitespace", + "variationType": "STRING" + }, + { + "description": "A non-string attribute (JSON number) does not match any SemVer condition and falls through to the default rather than erroring.", + "attributes": { + "app_version": 1.2 + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-string-attribute", + "variationType": "STRING" + }, + { + "description": "A flag with an invalid configured SemVer comparand returns a parse error while the rest of the configuration remains usable.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json new file mode 100644 index 00000000000..bac7fdade7b --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json @@ -0,0 +1,118 @@ +[ + { + "description": "A one-part version is normalized with missing minor and patch parts: \"18\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "one-part-input", + "app_version": "18" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-input" + }, + "targetingKey": "one-part-input", + "variationType": "STRING" + }, + { + "description": "A one-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-one-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-comparand" + }, + "targetingKey": "one-part-comparand", + "variationType": "STRING" + }, + { + "description": "A two-part version is normalized with a missing patch part: \"18.0\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "two-part-input", + "app_version": "18.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-input" + }, + "targetingKey": "two-part-input", + "variationType": "STRING" + }, + { + "description": "A two-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18.0\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-two-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-comparand" + }, + "targetingKey": "two-part-comparand", + "variationType": "STRING" + }, + { + "description": "A four-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "four-part-input", + "app_version": "18.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-input" + }, + "targetingKey": "four-part-input", + "variationType": "STRING" + }, + { + "description": "A four-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-four-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-comparand" + }, + "targetingKey": "four-part-comparand", + "variationType": "STRING" + }, + { + "description": "A five-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "five-part-input", + "app_version": "18.0.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-input" + }, + "targetingKey": "five-part-input", + "variationType": "STRING" + }, + { + "description": "A five-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-five-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-comparand" + }, + "targetingKey": "five-part-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json new file mode 100644 index 00000000000..8bfadabb0e7 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json @@ -0,0 +1,97 @@ +[ + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-zero-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 0, + "doLog": true + } + }, + { + "flag": "serial-id-missing-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": "control", + "reason": "STATIC", + "serialId": null, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-vip", + "attributes": { + "segment": "vip" + }, + "result": { + "value": "vip", + "reason": "TARGETING_MATCH", + "serialId": 201, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-standard", + "attributes": { + "segment": "standard" + }, + "result": { + "value": "standard", + "reason": "STATIC", + "serialId": 200, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json new file mode 100644 index 00000000000..dd988ba8583 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json @@ -0,0 +1,67 @@ +[ + { + "flag": "basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 100, + "doLog": true + } + }, + { + "flag": "experiment-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": "treatment", + "reason": "STATIC", + "serialId": 108, + "doLog": true + } + }, + { + "flag": "no-log-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-no-log", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 200, + "doLog": false + } + }, + { + "flag": "multi-serial-flag-1", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 1, + "reason": "STATIC", + "serialId": 128, + "doLog": true + } + }, + { + "flag": "multi-serial-flag-2", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 2, + "reason": "STATIC", + "serialId": 130, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json similarity index 65% rename from tests/parametric/test_ffe/test-case-start-and-end-date-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json index 7a48ec35886..53bf8859cdd 100644 --- a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "version": "1.15.0", - "country": "US" + "country": "US", + "version": "1.15.0" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "version": "0.20.1", - "country": "Canada" + "country": "Canada", + "version": "0.20.1" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "version": "2.1.13" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json new file mode 100644 index 00000000000..1bc2c914534 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json @@ -0,0 +1,16 @@ +[ + { + "description": "The flag and surrounding UFC config include unknown object fields that SDKs must ignore while evaluating the known schema fields.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "unknown-fields-tolerance-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "on" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json new file mode 100644 index 00000000000..c37fde4d573 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json @@ -0,0 +1,31 @@ +[ + { + "description": "The first allocation uses an unknown operator and the second is a catch-all trap. SDKs must return a parse error without falling through to the catch-all allocation.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "operator-grease-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the unknown-operator flag is skipped.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json new file mode 100644 index 00000000000..ca3b775ad61 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json @@ -0,0 +1,27 @@ +[ + { + "description": "A flag whose variation violates its declared type returns a parse error while the rest of the configuration remains usable.", + "attributes": {}, + "defaultValue": 0, + "flag": "variant-type-mismatch-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "user-1", + "variationType": "INTEGER" + }, + { + "description": "A valid flag in the same configuration continues to evaluate when a malformed neighboring flag returns a parse error.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json new file mode 100644 index 00000000000..0be74f1f7ac --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json @@ -0,0 +1,46 @@ +[ + { + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "alice", + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "bob", + "variationType": "NUMERIC" + }, + { + "attributes": { + "age": 50 + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" + } +] diff --git a/tests/parametric/test_ffe/test-json-config-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json similarity index 62% rename from tests/parametric/test_ffe/test-json-config-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json index ecc799546b0..45a9c2ee973 100644 --- a/tests/parametric/test_ffe/test-json-config-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json @@ -1,72 +1,76 @@ [ { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 1.0, "integer": 1, - "string": "one", - "float": 1.0 + "string": "one" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "diana", "attributes": { "Force Empty": true }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "TARGETING_MATCH", "value": {} - } + }, + "targetingKey": "diana", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json similarity index 70% rename from tests/parametric/test_ffe/test-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json index 45867e5897c..2b03b9a39a1 100644 --- a/tests/parametric/test_ffe/test-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "message": "Hello, world!" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "message": "Hello, world!" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json index 120647ec3b5..1ed7795bb95 100644 --- a/tests/parametric/test_ffe/test-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json @@ -1,54 +1,58 @@ [ { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ash", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "kümmert", "b": "schön" } - } + }, + "targetingKey": "ash", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ben", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "піклуватися", "b": "любов" } - } + }, + "targetingKey": "ben", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "cameron", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "照顾", "b": "漂亮" } - } + }, + "targetingKey": "cameron", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "darryl", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "🤗", "b": "🌸" } - } + }, + "targetingKey": "darryl", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-string-with-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-string-with-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json index e56322d44f0..32397a48398 100644 --- a/tests/parametric/test_ffe/test-string-with-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json @@ -1,794 +1,860 @@ [ { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_spaces", "attributes": { "string_with_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " a b c d e f " - } + }, + "targetingKey": "string_with_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_space", "attributes": { "string_with_only_one_space": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_one_space", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_spaces", "attributes": { "string_with_only_multiple_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_multiple_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dots", "attributes": { "string_with_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ".a.b.c.d.e.f." - } + }, + "targetingKey": "string_with_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dot", "attributes": { "string_with_only_one_dot": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "." - } + }, + "targetingKey": "string_with_only_one_dot", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dots", "attributes": { "string_with_only_multiple_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "......." - } + }, + "targetingKey": "string_with_only_multiple_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_comas", "attributes": { "string_with_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",a,b,c,d,e,f," - } + }, + "targetingKey": "string_with_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_coma", "attributes": { "string_with_only_one_coma": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "," - } + }, + "targetingKey": "string_with_only_one_coma", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_comas", "attributes": { "string_with_only_multiple_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",,,,,,," - } + }, + "targetingKey": "string_with_only_multiple_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_colons", "attributes": { "string_with_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":a:b:c:d:e:f:" - } + }, + "targetingKey": "string_with_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_colon", "attributes": { "string_with_only_one_colon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":" - } + }, + "targetingKey": "string_with_only_one_colon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_colons", "attributes": { "string_with_only_multiple_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":::::::" - } + }, + "targetingKey": "string_with_only_multiple_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_semicolons", "attributes": { "string_with_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";a;b;c;d;e;f;" - } + }, + "targetingKey": "string_with_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_semicolon", "attributes": { "string_with_only_one_semicolon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";" - } + }, + "targetingKey": "string_with_only_one_semicolon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_semicolons", "attributes": { "string_with_only_multiple_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";;;;;;;" - } + }, + "targetingKey": "string_with_only_multiple_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_slashes", "attributes": { "string_with_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/a/b/c/d/e/f/" - } + }, + "targetingKey": "string_with_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_slash", "attributes": { "string_with_only_one_slash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/" - } + }, + "targetingKey": "string_with_only_one_slash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_slashes", "attributes": { "string_with_only_multiple_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "///////" - } + }, + "targetingKey": "string_with_only_multiple_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dashes", "attributes": { "string_with_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-a-b-c-d-e-f-" - } + }, + "targetingKey": "string_with_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dash", "attributes": { "string_with_only_one_dash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-" - } + }, + "targetingKey": "string_with_only_one_dash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dashes", "attributes": { "string_with_only_multiple_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-------" - } + }, + "targetingKey": "string_with_only_multiple_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_underscores", "attributes": { "string_with_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_a_b_c_d_e_f_" - } + }, + "targetingKey": "string_with_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_underscore", "attributes": { "string_with_only_one_underscore": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_" - } + }, + "targetingKey": "string_with_only_one_underscore", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_underscores", "attributes": { "string_with_only_multiple_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_______" - } + }, + "targetingKey": "string_with_only_multiple_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_plus_signs", "attributes": { "string_with_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+a+b+c+d+e+f+" - } + }, + "targetingKey": "string_with_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_plus_sign", "attributes": { "string_with_only_one_plus_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+" - } + }, + "targetingKey": "string_with_only_one_plus_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_plus_signs", "attributes": { "string_with_only_multiple_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+++++++" - } + }, + "targetingKey": "string_with_only_multiple_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_equal_signs", "attributes": { "string_with_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=a=b=c=d=e=f=" - } + }, + "targetingKey": "string_with_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_equal_sign", "attributes": { "string_with_only_one_equal_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=" - } + }, + "targetingKey": "string_with_only_one_equal_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_equal_signs", "attributes": { "string_with_only_multiple_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=======" - } + }, + "targetingKey": "string_with_only_multiple_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dollar_signs", "attributes": { "string_with_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$a$b$c$d$e$f$" - } + }, + "targetingKey": "string_with_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dollar_sign", "attributes": { "string_with_only_one_dollar_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$" - } + }, + "targetingKey": "string_with_only_one_dollar_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dollar_signs", "attributes": { "string_with_only_multiple_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$$$$$$$" - } + }, + "targetingKey": "string_with_only_multiple_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_at_signs", "attributes": { "string_with_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@a@b@c@d@e@f@" - } + }, + "targetingKey": "string_with_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_at_sign", "attributes": { "string_with_only_one_at_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@" - } + }, + "targetingKey": "string_with_only_one_at_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_at_signs", "attributes": { "string_with_only_multiple_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@@@@@@@" - } + }, + "targetingKey": "string_with_only_multiple_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_amp_signs", "attributes": { "string_with_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&a&b&c&d&e&f&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026a\u0026b\u0026c\u0026d\u0026e\u0026f\u0026" + }, + "targetingKey": "string_with_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_amp_sign", "attributes": { "string_with_only_one_amp_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026" + }, + "targetingKey": "string_with_only_one_amp_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_amp_signs", "attributes": { "string_with_only_multiple_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&&&&&&&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026\u0026\u0026\u0026\u0026\u0026\u0026" + }, + "targetingKey": "string_with_only_multiple_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_hash_signs", "attributes": { "string_with_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#a#b#c#d#e#f#" - } + }, + "targetingKey": "string_with_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_hash_sign", "attributes": { "string_with_only_one_hash_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#" - } + }, + "targetingKey": "string_with_only_one_hash_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_hash_signs", "attributes": { "string_with_only_multiple_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#######" - } + }, + "targetingKey": "string_with_only_multiple_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_percentage_signs", "attributes": { "string_with_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%a%b%c%d%e%f%" - } + }, + "targetingKey": "string_with_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_percentage_sign", "attributes": { "string_with_only_one_percentage_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%" - } + }, + "targetingKey": "string_with_only_one_percentage_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_percentage_signs", "attributes": { "string_with_only_multiple_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%%%%%%%" - } + }, + "targetingKey": "string_with_only_multiple_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_tilde_signs", "attributes": { "string_with_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~a~b~c~d~e~f~" - } + }, + "targetingKey": "string_with_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_tilde_sign", "attributes": { "string_with_only_one_tilde_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~" - } + }, + "targetingKey": "string_with_only_one_tilde_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_tilde_signs", "attributes": { "string_with_only_multiple_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~~~~~~~" - } + }, + "targetingKey": "string_with_only_multiple_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_asterix_signs", "attributes": { "string_with_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*a*b*c*d*e*f*" - } + }, + "targetingKey": "string_with_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_asterix_sign", "attributes": { "string_with_only_one_asterix_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*" - } + }, + "targetingKey": "string_with_only_one_asterix_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_asterix_signs", "attributes": { "string_with_only_multiple_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*******" - } + }, + "targetingKey": "string_with_only_multiple_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_single_quotes", "attributes": { "string_with_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'a'b'c'd'e'f'" - } + }, + "targetingKey": "string_with_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_single_quote", "attributes": { "string_with_only_one_single_quote": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'" - } + }, + "targetingKey": "string_with_only_one_single_quote", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_single_quotes", "attributes": { "string_with_only_multiple_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'''''''" - } + }, + "targetingKey": "string_with_only_multiple_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_question_marks", "attributes": { "string_with_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?a?b?c?d?e?f?" - } + }, + "targetingKey": "string_with_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_question_mark", "attributes": { "string_with_only_one_question_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?" - } + }, + "targetingKey": "string_with_only_one_question_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_question_marks", "attributes": { "string_with_only_multiple_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "???????" - } + }, + "targetingKey": "string_with_only_multiple_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_exclamation_marks", "attributes": { "string_with_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!a!b!c!d!e!f!" - } + }, + "targetingKey": "string_with_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_exclamation_mark", "attributes": { "string_with_only_one_exclamation_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!" - } + }, + "targetingKey": "string_with_only_one_exclamation_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_exclamation_marks", "attributes": { "string_with_only_multiple_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!!!!!!!" - } + }, + "targetingKey": "string_with_only_multiple_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_opening_parentheses", "attributes": { "string_with_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(a(b(c(d(e(f(" - } + }, + "targetingKey": "string_with_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_opening_parenthese", "attributes": { "string_with_only_one_opening_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(" - } + }, + "targetingKey": "string_with_only_one_opening_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_opening_parentheses", "attributes": { "string_with_only_multiple_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(((((((" - } + }, + "targetingKey": "string_with_only_multiple_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_closing_parentheses", "attributes": { "string_with_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")a)b)c)d)e)f)" - } + }, + "targetingKey": "string_with_closing_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_closing_parenthese", "attributes": { "string_with_only_one_closing_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")" - } + }, + "targetingKey": "string_with_only_one_closing_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_closing_parentheses", "attributes": { "string_with_only_multiple_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")))))))" - } + }, + "targetingKey": "string_with_only_multiple_closing_parentheses", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/flags-v1.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json similarity index 60% rename from tests/parametric/test_ffe/flags-v1.json rename to tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index f01bfe2ef86..a436b889b15 100644 --- a/tests/parametric/test_ffe/flags-v1.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -2,8 +2,11 @@ "createdAt": "2024-04-17T19:40:53.716Z", "format": "SERVER", "environment": { - "name": "Test" + "name": "Test", + "unknownEnvironmentField": "ignored" }, + "unknownTopLevelField": "ignored", + "unknownTopLevelNull": null, "flags": { "empty_flag": { "key": "empty_flag", @@ -78,9 +81,109 @@ "test": { "key": "test", "value": "test" + }, + "capturing-groups": { + "key": "capturing-groups", + "value": "capturing-groups" + }, + "unicode-word-boundary": { + "key": "unicode-word-boundary", + "value": "unicode-word-boundary" + }, + "comma-quantifier": { + "key": "comma-quantifier", + "value": "comma-quantifier" + }, + "literal-whitespace": { + "key": "literal-whitespace", + "value": "literal-whitespace" } }, "allocations": [ + { + "key": "comma-quantifier", + "rules": [ + { + "conditions": [ + { + "attribute": "comma_regex_case", + "operator": "MATCHES", + "value": "a{1,3}" + } + ] + } + ], + "splits": [ + { + "variationKey": "comma-quantifier", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "literal-whitespace", + "rules": [ + { + "conditions": [ + { + "attribute": "whitespace_regex_case", + "operator": "MATCHES", + "value": " exact " + } + ] + } + ], + "splits": [ + { + "variationKey": "literal-whitespace", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "capturing-groups", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "^([[:alnum:]._%+-]+)@capture\\.example$" + } + ] + } + ], + "splits": [ + { + "variationKey": "capturing-groups", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "unicode-word-boundary", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "(?u)\\bmañana\\b" + } + ] + } + ], + "splits": [ + { + "variationKey": "unicode-word-boundary", + "shards": [] + } + ], + "doLog": true + }, { "key": "partial-example", "rules": [ @@ -612,6 +715,279 @@ } ] }, + "semver-comparison-test": { + "key": "semver-comparison-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "legacy": { + "key": "legacy", + "value": "legacy" + }, + "stable": { + "key": "stable", + "value": "stable" + }, + "beta": { + "key": "beta", + "value": "beta" + }, + "launch": { + "key": "launch", + "value": "launch" + }, + "future": { + "key": "future", + "value": "future" + }, + "maximum": { + "key": "maximum", + "value": "maximum" + } + }, + "allocations": [ + { + "key": "maximum-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "9007199254740991.9007199254740991.9007199254740991" + } + ] + } + ], + "splits": [ + { + "variationKey": "maximum", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "legacy-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "1.2.3-alpha.4" + } + ] + }, + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "1.2.3-alpha.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "2.3.4" + } + ] + } + ], + "splits": [ + { + "variationKey": "legacy", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "stable-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "2.3.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "3.4.5-beta.2" + } + ] + } + ], + "splits": [ + { + "variationKey": "stable", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "beta-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "3.4.5-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "4.5.6-rc.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "launch-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "4.5.6-rc.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "launch", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "future-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "4.5.6-rc.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "5.6.7" + } + ] + } + ], + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-test": { + "key": "semver-invalid-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an invalid configured SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-waterfall-test": { + "key": "semver-invalid-comparand-waterfall-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately places an invalid configured SemVer comparand in a non-last allocation of a waterfall, followed by a valid default allocation. SDKs must reject the whole flag during parsing rather than skipping the invalid allocation and falling through to the default.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + }, + "fallback": { + "key": "fallback", + "value": "fallback" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "default", + "rules": [], + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, "start-and-end-date-test": { "key": "start-and-end-date-test", "enabled": true, @@ -3293,6 +3669,1739 @@ "doLog": true } ] + }, + "microsecond-date-test": { + "key": "microsecond-date-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "expired": { + "key": "expired", + "value": "expired" + }, + "active": { + "key": "active", + "value": "active" + }, + "future": { + "key": "future", + "value": "future" + } + }, + "allocations": [ + { + "key": "expired-allocation", + "splits": [ + { + "variationKey": "expired", + "shards": [] + } + ], + "endAt": "2002-10-31T09:00:00.594321Z", + "doLog": true + }, + { + "key": "future-allocation", + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "startAt": "2052-10-31T09:00:00.123456Z", + "doLog": true + }, + { + "key": "active-allocation", + "splits": [ + { + "variationKey": "active", + "shards": [] + } + ], + "startAt": "2022-10-31T09:00:00.235982Z", + "endAt": "2050-10-31T09:00:00.987654Z", + "doLog": true + } + ] + }, + "unknown-fields-tolerance-flag": { + "key": "unknown-fields-tolerance-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately contains unknown fields at every fixed-schema UFC level. SDKs must ignore those fields and evaluate the flag normally.", + "unknownFlagField": "ignored", + "variations": { + "on": { + "key": "on", + "value": "on", + "unknownVariationField": "ignored" + } + }, + "allocations": [ + { + "key": "unknown-fields-allocation", + "unknownAllocationField": "ignored", + "rules": [ + { + "unknownRuleField": "ignored", + "conditions": [ + { + "attribute": "country", + "operator": "ONE_OF", + "value": [ + "US" + ], + "unknownConditionField": "ignored" + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "unknownSplitField": "ignored", + "shards": [ + { + "salt": "unknown-fields-tolerance-flag", + "totalShards": 10000, + "unknownShardField": "ignored", + "ranges": [ + { + "start": 0, + "end": 10000, + "unknownRangeField": "ignored" + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "operator-grease-flag": { + "key": "operator-grease-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an unknown condition operator. SDKs must remove this flag rather than falling through to the catch-all allocation or rejecting the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": [ + { + "key": "grease-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": "NOT_A_REAL_OPERATOR", + "value": "anything" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-regex-flag": { + "key": "invalid-regex-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses the invalid regex *@datadoghq.com. SDKs must remove this flag rather than falling through to a later allocation or rejecting the whole config.", + "variations": { + "invalid-regex-match": { + "key": "invalid-regex-match", + "value": "invalid-regex-match" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-regex-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "*@datadoghq.com" + } + ] + } + ], + "splits": [ + { + "variationKey": "invalid-regex-match", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-regex-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "variant-type-mismatch-flag": { + "key": "variant-type-mismatch-flag", + "enabled": true, + "variationType": "INTEGER", + "comment": "This flag is intentionally invalid: one variation contains a string even though the flag declares INTEGER values. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "invalid": { + "key": "invalid", + "value": "not-an-integer" + }, + "valid": { + "key": "valid", + "value": 0 + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "invalid", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-gt-condition-flag": { + "key": "invalid-gt-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives GT a string operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-gt-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "age", + "operator": "GT", + "value": "not-a-number" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-gt-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-one-of-condition-flag": { + "key": "invalid-one-of-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives ONE_OF a scalar operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-one-of-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "role", + "operator": "ONE_OF", + "value": "admin" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-one-of-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-is-null-condition-flag": { + "key": "invalid-is-null-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives IS_NULL a non-boolean operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-is-null-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "deleted_at", + "operator": "IS_NULL", + "value": "not-a-boolean" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-is-null-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "malformed-allocations-flag": { + "key": "malformed-allocations-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: allocations must be an array. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": "this-is-not-a-list" + }, + "missing-split-shards-flag": { + "key": "missing-split-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards is required even when the intended split is unsharded. SDKs must treat omitted shards differently from an explicit empty shards array.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-split-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap" + } + ], + "doLog": true + } + ] + }, + "overflow-total-shards-flag": { + "key": "overflow-total-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].totalShards must fit in an unsigned 32-bit integer. SDKs must remove this flag rather than overflowing or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "overflow-total-shards-flag", + "totalShards": 4294967296, + "ranges": [ + { + "start": 0, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "negative-shard-range-start-flag": { + "key": "negative-shard-range-start-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[].start must be an unsigned integer. SDKs must remove this flag rather than treating a negative range as matching traffic.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "negative-shard-range-start-flag", + "totalShards": 100, + "ranges": [ + { + "start": -1, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "null-shard-range-flag": { + "key": "null-shard-range-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[] contains null. SDKs must remove this flag rather than panicking or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "null-shard-range-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "null-shard-range-flag", + "totalShards": 8192, + "ranges": [ + null + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "null-shard-range-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "valid-flag-after-invalid-config": { + "key": "valid-flag-after-invalid-config", + "enabled": true, + "variationType": "STRING", + "comment": "This valid neighbor proves malformed or unsupported flags do not poison the rest of the remote config.", + "variations": { + "expected": { + "key": "expected", + "value": "expected" + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "expected", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-precedence-test": { + "key": "semver-precedence-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates SemVer precedence rules that are easy to get wrong: multi-digit numeric prerelease ordering (beta.2 < beta.11), build-metadata-ignored across every operator (EQ/NEQ/LT/LTE/GT/GTE), and alphanumeric prerelease ordering (alpha < alpha.1 < alpha.beta < beta < release). Each allocation gates on a unique SEMVER_EQ attribute version so test attributes cannot cross-match other allocations; negative build-metadata cases (NEQ/LT/GT) fall through to DEFAULT.", + "variations": { + "numeric-prerelease": { + "key": "numeric-prerelease", + "value": "numeric-prerelease" + }, + "build-equal-precedence": { + "key": "build-equal-precedence", + "value": "build-equal-precedence" + }, + "build-dotted-eq": { + "key": "build-dotted-eq", + "value": "build-dotted-eq" + }, + "build-lte": { + "key": "build-lte", + "value": "build-lte" + }, + "build-gte": { + "key": "build-gte", + "value": "build-gte" + }, + "alpha-before-alpha-1": { + "key": "alpha-before-alpha-1", + "value": "alpha-before-alpha-1" + }, + "alpha-1-before-alpha-beta": { + "key": "alpha-1-before-alpha-beta", + "value": "alpha-1-before-alpha-beta" + }, + "alpha-beta-before-beta": { + "key": "alpha-beta-before-beta", + "value": "alpha-beta-before-beta" + }, + "beta-before-release": { + "key": "beta-before-release", + "value": "beta-before-release" + }, + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "numeric-prerelease", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "10.0.0-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "10.0.0-beta.11" + } + ] + } + ], + "splits": [ + { + "variationKey": "numeric-prerelease", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-equal-precedence", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+linux" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+darwin" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-equal-precedence", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-dotted-eq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0+exp.sha.5114f85" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-dotted-eq", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "40.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "40.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-lte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "50.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "50.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-gte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-neq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "60.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "60.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "70.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "70.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "80.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "80.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-before-alpha-1", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "90.0.0-alpha" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "90.0.0-alpha.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-before-alpha-1", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-1-before-alpha-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "91.0.0-alpha.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "91.0.0-alpha.beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-1-before-alpha-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-beta-before-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "92.0.0-alpha.beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "92.0.0-beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-beta-before-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "beta-before-release", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "93.0.0-beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "93.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "beta-before-release", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-input-test": { + "key": "semver-version-parts-input-test", + "enabled": true, + "variationType": "STRING", + "comment": "Exercises one- through five-part runtime attributes using only canonical three-part configured comparands, so comparand ingestion cannot prevent an input case from running. Scenario gates isolate each runtime comparison.", + "variations": { + "one-part-input": { + "key": "one-part-input", + "value": "one-part-input" + }, + "two-part-input": { + "key": "two-part-input", + "value": "two-part-input" + }, + "four-part-input": { + "key": "four-part-input", + "value": "four-part-input" + }, + "five-part-input": { + "key": "five-part-input", + "value": "five-part-input" + } + }, + "allocations": [ + { + "key": "one-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "one-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "two-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "two-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "four-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "four-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "five-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "five-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-input", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-one-part-comparand-test": { + "key": "semver-version-parts-one-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a one-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "one-part-comparand": { + "key": "one-part-comparand", + "value": "one-part-comparand" + } + }, + "allocations": [ + { + "key": "one-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-two-part-comparand-test": { + "key": "semver-version-parts-two-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a two-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "two-part-comparand": { + "key": "two-part-comparand", + "value": "two-part-comparand" + } + }, + "allocations": [ + { + "key": "two-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-four-part-comparand-test": { + "key": "semver-version-parts-four-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a four-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "four-part-comparand": { + "key": "four-part-comparand", + "value": "four-part-comparand" + } + }, + "allocations": [ + { + "key": "four-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-five-part-comparand-test": { + "key": "semver-version-parts-five-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a five-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "five-part-comparand": { + "key": "five-part-comparand", + "value": "five-part-comparand" + } + }, + "allocations": [ + { + "key": "five-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-syntax-test": { + "key": "semver-invalid-comparand-syntax-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-parseable SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "not-a-version" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-valid-comparand-hyphenated-prerelease-test": { + "key": "semver-valid-comparand-hyphenated-prerelease-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag uses a valid SemVer comparand with repeated hyphens inside its prerelease identifiers.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "valid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3----R-S.12.9.1--.12+meta" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-consecutive-dots-test": { + "key": "semver-invalid-comparand-consecutive-dots-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses consecutive dots in its SemVer prerelease comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.0.0-alpha..1" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-dot-suffix-test": { + "key": "semver-invalid-comparand-dot-suffix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a Ruby-style dot-suffixed SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3.DEV" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-vprefix-test": { + "key": "semver-invalid-comparand-vprefix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a v-prefixed SemVer comparand, which is not valid SemVer. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "v1.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-leading-zero-test": { + "key": "semver-invalid-comparand-leading-zero-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a leading-zero core component in its SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "01.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-non-string-test": { + "key": "semver-invalid-comparand-non-string-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-string (JSON number) SemVer comparand. SDKs must reject this as an invalid comparand without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": 1.2 + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-basic-flag": { + "key": "serial-id-basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 42 + } + ], + "doLog": true + } + ] + }, + "serial-id-zero-flag": { + "key": "serial-id-zero-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "zero-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 0 + } + ], + "doLog": true + } + ] + }, + "serial-id-missing-flag": { + "key": "serial-id-missing-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "no-serial-allocation", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-targeting-flag": { + "key": "serial-id-targeting-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "vip": { + "key": "vip", + "value": "vip" + }, + "standard": { + "key": "standard", + "value": "standard" + } + }, + "allocations": [ + { + "key": "vip-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "segment", + "operator": "ONE_OF", + "value": [ + "vip" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "vip", + "shards": [], + "serialId": 201 + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "standard", + "shards": [], + "serialId": 200 + } + ], + "doLog": true + } + ] + }, + "basic-flag": { + "key": "basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 100 + } + ], + "doLog": true + } + ] + }, + "experiment-flag": { + "key": "experiment-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "experiment-allocation", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 108 + } + ], + "doLog": true + } + ] + }, + "no-log-flag": { + "key": "no-log-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "no-log-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 200 + } + ], + "doLog": false + } + ] + }, + "multi-serial-flag-1": { + "key": "multi-serial-flag-1", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v1": { + "key": "v1", + "value": 1 + } + }, + "allocations": [ + { + "key": "allocation-1", + "rules": [], + "splits": [ + { + "variationKey": "v1", + "shards": [], + "serialId": 128 + } + ], + "doLog": true + } + ] + }, + "multi-serial-flag-2": { + "key": "multi-serial-flag-2", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v2": { + "key": "v2", + "value": 2 + } + }, + "allocations": [ + { + "key": "allocation-2", + "rules": [], + "splits": [ + { + "variationKey": "v2", + "shards": [], + "serialId": 130 + } + ], + "doLog": true + } + ] } } -} \ No newline at end of file +} diff --git a/tests/parametric/test_ffe/serial-id-metadata-flags.json b/tests/parametric/test_ffe/serial-id-metadata-flags.json deleted file mode 100644 index 35bf0ab4c74..00000000000 --- a/tests/parametric/test_ffe/serial-id-metadata-flags.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "createdAt": "2024-01-01T00:00:00.000Z", - "format": "SERVER", - "environment": { "name": "Test" }, - "flags": { - "serial-id-basic-flag": { - "key": "serial-id-basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 42 } - ], - "doLog": true - } - ] - }, - "serial-id-zero-flag": { - "key": "serial-id-zero-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "zero-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 0 } - ], - "doLog": true - } - ] - }, - "serial-id-missing-flag": { - "key": "serial-id-missing-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { "key": "control", "value": "control" }, - "treatment": { "key": "treatment", "value": "treatment" } - }, - "allocations": [ - { - "key": "no-serial-allocation", - "rules": [], - "splits": [ - { "variationKey": "control", "shards": [] } - ], - "doLog": true - } - ] - }, - "serial-id-targeting-flag": { - "key": "serial-id-targeting-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "vip": { "key": "vip", "value": "vip" }, - "standard": { "key": "standard", "value": "standard" } - }, - "allocations": [ - { - "key": "vip-allocation", - "rules": [ - { - "conditions": [ - { "attribute": "segment", "operator": "ONE_OF", "value": ["vip"] } - ] - } - ], - "splits": [ - { "variationKey": "vip", "shards": [], "serialId": 201 } - ], - "doLog": true - }, - { - "key": "default-allocation", - "rules": [], - "splits": [ - { "variationKey": "standard", "shards": [], "serialId": 200 } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/span-enrichment-flags.json b/tests/parametric/test_ffe/span-enrichment-flags.json deleted file mode 100644 index 232315bcbe8..00000000000 --- a/tests/parametric/test_ffe/span-enrichment-flags.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "createdAt": "2024-04-17T19:40:53.716Z", - "format": "SERVER", - "environment": { - "name": "Test" - }, - "flags": { - "basic-flag": { - "key": "basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 100 - } - ], - "doLog": true - } - ] - }, - "experiment-flag": { - "key": "experiment-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { - "key": "control", - "value": "control" - }, - "treatment": { - "key": "treatment", - "value": "treatment" - } - }, - "allocations": [ - { - "key": "experiment-allocation", - "rules": [], - "splits": [ - { - "variationKey": "treatment", - "shards": [], - "serialId": 108 - } - ], - "doLog": true - } - ] - }, - "no-log-flag": { - "key": "no-log-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "no-log-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 200 - } - ], - "doLog": false - } - ] - }, - "multi-serial-flag-1": { - "key": "multi-serial-flag-1", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v1": { - "key": "v1", - "value": 1 - } - }, - "allocations": [ - { - "key": "allocation-1", - "rules": [], - "splits": [ - { - "variationKey": "v1", - "shards": [], - "serialId": 128 - } - ], - "doLog": true - } - ] - }, - "multi-serial-flag-2": { - "key": "multi-serial-flag-2", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v2": { - "key": "v2", - "value": 2 - } - }, - "allocations": [ - { - "key": "allocation-2", - "rules": [], - "splits": [ - { - "variationKey": "v2", - "shards": [], - "serialId": 130 - } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/test-case-regex-flag.json b/tests/parametric/test_ffe/test-case-regex-flag.json deleted file mode 100644 index 94aa87f23a9..00000000000 --- a/tests/parametric/test_ffe/test-case-regex-flag.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "alice", - "attributes": { - "version": "1.15.0", - "email": "alice@example.com" - }, - "result": { - "value": "partial-example" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "bob", - "attributes": { - "version": "0.20.1", - "email": "bob@test.com" - }, - "result": { - "value": "test" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "charlie", - "attributes": { - "version": "2.1.13" - }, - "result": { - "value": "none" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "derek", - "attributes": { - "version": "2.1.13", - "email": "derek@gmail.com" - }, - "result": { - "value": "none" - } - } -] diff --git a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/test-flag-that-does-not-exist.json deleted file mode 100644 index 7499bba1c50..00000000000 --- a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", - "attributes": { - "age": 50 - }, - "result": { - "value": 0.0 - } - } -] diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index 79a4a716b8f..ca533dc42bd 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -34,7 +34,7 @@ parametrize = pytest.mark.parametrize pytest_plugins = ["utils.mocked_backend.ffe"] -UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json" +UFC_VALID_FIXTURE = Path(__file__).parent / "ffe-system-test-data" / "ufc-config.json" RC_PRODUCT = "FFE_FLAGS" TEST_API_KEY = "system-tests-mock-api-key" MOCK_STATUS_ATTEMPTS = 25 diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 35b15215c50..642a73d56d0 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -18,6 +18,8 @@ RC_PATH = f"datadog/2/{RC_PRODUCT}" FFE_READY_RETRY_ATTEMPTS = 10 FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 +FFE_SYSTEM_TEST_DATA_DIR = Path(__file__).parent / "ffe-system-test-data" +FFE_EVALUATION_CASES_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-cases" parametrize = pytest.mark.parametrize @@ -25,10 +27,7 @@ # Load the UFC fixture file at module level def _load_ufc_fixture() -> dict[str, Any]: """Load the UFC fixture file.""" - fixture_path = Path(__file__).parent / "flags-v1.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") + fixture_path = FFE_SYSTEM_TEST_DATA_DIR / "ufc-config.json" with fixture_path.open() as f: return json.load(f) @@ -36,13 +35,10 @@ def _load_ufc_fixture() -> dict[str, Any]: def _get_test_case_files() -> list[str]: """Get all test case files from the fixtures directory.""" - test_data_dir = Path(__file__).parent - if not test_data_dir.exists(): - return [] - - # Exclude base fixtures that aren't test cases - excluded = {"flags-v1.json", "serial-id-metadata-flags.json", "span-enrichment-flags.json"} - return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] + case_files = sorted(path.name for path in FFE_EVALUATION_CASES_DIR.glob("*.json")) + if not case_files: + raise FileNotFoundError(f"No canonical evaluation cases found in {FFE_EVALUATION_CASES_DIR}") + return case_files # Load fixture at module level for reuse across tests @@ -103,7 +99,7 @@ def _ffe_evaluate_with_rc_retry( flag: str, variation_type: str, default_value: bool | str | float | dict[str, Any], - targeting_key: str, + targeting_key: str | None, attributes: dict[str, Any] | None = None, ) -> dict[str, Any]: result = test_library.ffe_evaluate( @@ -159,13 +155,12 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI """ # Load the test case file - test_case_path = Path(__file__).parent / test_case_file - - if not test_case_path.exists(): - pytest.skip(f"Test case file not found: {test_case_path}") + test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file with test_case_path.open() as f: test_cases = json.load(f) + assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" + assert test_cases, f"No evaluation cases in {test_case_file}" # Set up UFC Remote Config and wait for it to be applied _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) @@ -181,7 +176,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI default_value = test_case["defaultValue"] targeting_key = test_case["targetingKey"] attributes = test_case.get("attributes", {}) - expected_result = test_case["result"]["value"] + expected_result = test_case["result"] result = _ffe_evaluate_with_rc_retry( test_library, @@ -198,8 +193,19 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI actual_value = result.get("value") # Assert the evaluation result matches expected value - assert actual_value == expected_result, ( + assert actual_value == expected_result["value"], ( f"Test case {i} in {test_case_file} failed: " f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={actual_value}" + f"expected={expected_result}, actual={result}" + ) + assert result.get("reason") == expected_result["reason"], ( + f"Test case {i} in {test_case_file}: expected reason={expected_result['reason']}, result={result}" ) + if "errorCode" in expected_result: + assert result.get("errorCode") == expected_result["errorCode"], ( + f"Test case {i} in {test_case_file}: expected errorCode={expected_result['errorCode']}, result={result}" + ) + elif expected_result["reason"] != "ERROR": + assert result.get("errorCode") in {None, ""}, ( + f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" + ) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 8e8a9e14d45..ec0d15b6f3e 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -23,7 +23,6 @@ import json import time -from pathlib import Path from typing import Any from utils import pytest @@ -32,6 +31,7 @@ from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import load_ufc_fixture_for_cases RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" @@ -43,18 +43,7 @@ parametrize = pytest.mark.parametrize -def _load_serial_id_metadata_fixture() -> dict[str, Any]: - """Load the UFC fixture file for serial ID metadata tests.""" - fixture_path = Path(__file__).parent / "serial-id-metadata-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SERIAL_ID_METADATA_DATA = _load_serial_id_metadata_fixture() +UFC_SERIAL_ID_METADATA_DATA = load_ufc_fixture_for_cases("test-case-serial-id-metadata.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/test_span_enrichment.py b/tests/parametric/test_ffe/test_span_enrichment.py index 6a4e54bdacb..749913bb455 100644 --- a/tests/parametric/test_ffe/test_span_enrichment.py +++ b/tests/parametric/test_ffe/test_span_enrichment.py @@ -21,14 +21,18 @@ import json from utils import pytest -from pathlib import Path from typing import Any from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary -from tests.parametric.test_ffe.utils import decode_delta_varint, encode_delta_varint, hash_targeting_key +from tests.parametric.test_ffe.utils import ( + decode_delta_varint, + encode_delta_varint, + hash_targeting_key, + load_ufc_fixture_for_cases, +) RC_PRODUCT = "FFE_FLAGS" @@ -37,18 +41,7 @@ parametrize = pytest.mark.parametrize -def _load_span_enrichment_fixture() -> dict[str, Any]: - """Load the UFC fixture file for span enrichment tests.""" - fixture_path = Path(__file__).parent / "span-enrichment-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SPAN_ENRICHMENT_DATA = _load_span_enrichment_fixture() +UFC_SPAN_ENRICHMENT_DATA = load_ufc_fixture_for_cases("test-case-span-enrichment-inputs.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index 325c825f949..ad655f9cf84 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -11,12 +11,26 @@ import base64 import hashlib +import json +from pathlib import Path +from typing import Any # Varint encoding constants VARINT_DATA_BITS = 0x7F # Lower 7 bits contain data VARINT_CONTINUATION_BIT = 0x80 # MSB indicates more bytes follow +def load_ufc_fixture_for_cases(case_filename: str) -> dict[str, Any]: + """Load the canonical flag definitions used by a specific integration suite.""" + fixture_root = Path(__file__).parent / "ffe-system-test-data" + config = json.loads((fixture_root / "ufc-config.json").read_text()) + cases = json.loads((fixture_root / "evaluation-cases" / case_filename).read_text()) + # Keep unrelated evaluator edge cases out of transport/span integration tests. + flag_keys = dict.fromkeys(case["flag"] for case in cases) + config["flags"] = {key: config["flags"][key] for key in flag_keys} + return config + + def decode_varint(data: bytes, offset: int = 0) -> tuple[int, int]: """Decode a single varint from bytes. diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py new file mode 100644 index 00000000000..8f53acd82ee --- /dev/null +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -0,0 +1,145 @@ +# ruff: noqa: SLF001 + +from pathlib import Path +import shutil + +from utils import pytest + +from utils import scenarios +from utils.scripts import update_ffe_fixtures + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["main", "feature/fixture-update", "v1.2.3", "ea8b5cc5"]) +def test_validate_fixture_ref_accepts_git_refs(fixture_ref: str) -> None: + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["", "-main", "../main", "main^", "main ref"]) +def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: + with pytest.raises(ValueError, match="Invalid FFE fixture ref"): + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + (source / "evaluation-cases").mkdir(parents=True) + snapshot.mkdir() + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + (source / "README.md").write_text("upstream documentation", encoding="utf-8") + (source / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") + (source / "precomputed-assignments").mkdir() + + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + assert (snapshot / "ufc-config.json").is_file() + assert (snapshot / "evaluation-cases" / "case.json").is_file() + assert not (snapshot / "README.md").exists() + assert not (snapshot / "AGENTS.md").exists() + assert not (snapshot / "precomputed-assignments").exists() + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + source.mkdir() + snapshot.mkdir() + (source / "evaluation-cases").mkdir() + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + target = source / "target.json" + target.write_text("{}", encoding="utf-8") + (source / "evaluation-cases" / "linked.json").symlink_to(target) + + with pytest.raises(ValueError, match="Refusing to copy symbolic link"): + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + +@scenarios.test_the_test +def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: + cases_directory = tmp_path / "evaluation-cases" + cases_directory.mkdir() + (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") + (cases_directory / "first.json").write_text('[{"flag": "one"}]', encoding="utf-8") + (cases_directory / "second.json").write_text('[{"flag": "two"}, {"flag": "three"}]', encoding="utf-8") + + assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 + + +@scenarios.test_the_test +def test_have_same_contents_ignores_generated_source_metadata(tmp_path: Path) -> None: + snapshot = tmp_path / "snapshot" + destination = tmp_path / "destination" + snapshot.mkdir() + destination.mkdir() + (snapshot / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "SOURCE.md").write_text("generated", encoding="utf-8") + + assert update_ffe_fixtures._have_same_contents(snapshot, destination) + + (destination / "fixture.json").write_text('{"changed": true}', encoding="utf-8") + assert not update_ffe_fixtures._have_same_contents(snapshot, destination) + + +@scenarios.test_the_test +def test_update_fixtures_preserves_snapshot_on_validation_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + destination.mkdir() + (destination / "SOURCE.md").write_text("original snapshot", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + with pytest.raises(ValueError, match="at least one evaluation case"): + update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_text() == "original snapshot" + + +@scenarios.test_the_test +def test_update_fixtures_records_source_and_removes_stale_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + output = tmp_path / "outputs" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + case_file = source / "evaluation-cases" / "case.json" + case_file.write_text('[{"flag": "flag"}]', encoding="utf-8") + destination.mkdir() + (destination / "stale.json").write_text("[]", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + assert update_ffe_fixtures.update_fixtures("main") + assert not (destination / "stale.json").exists() + assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() + metadata = (destination / "SOURCE.md").read_bytes() + assert ("a" * 40).encode() in metadata + assert "fixture_count=1\nchanged=true\n" in output.read_text() + + (source / "README.md").write_text("documentation-only upstream change", encoding="utf-8") + assert not update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_bytes() == metadata + assert output.read_text().endswith("changed=false\n") diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index 5a512b87c19..895f006e227 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -824,7 +824,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: @@ -1267,7 +1267,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: diff --git a/utils/mocked_backend/ffe.py b/utils/mocked_backend/ffe.py index 5b288d176e2..dbb1314a5f4 100644 --- a/utils/mocked_backend/ffe.py +++ b/utils/mocked_backend/ffe.py @@ -48,7 +48,7 @@ EXPECTED_DD_ENV = "test" CONFIG_QUERY = urlencode({"dd_env": EXPECTED_DD_ENV}) REPO_ROOT = Path(__file__).parents[2] -UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "flags-v1.json" +UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" / "ufc-config.json" MALFORMED_UFC_BYTES = b'{"flags": [' UFC_RESPONSE_TYPE = "universal-flag-configuration" diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py new file mode 100644 index 00000000000..e7f3b95f484 --- /dev/null +++ b/utils/scripts/update_ffe_fixtures.py @@ -0,0 +1,206 @@ +"""Refresh the checked-in FFE fixture snapshot from its canonical repository.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DESTINATION = REPOSITORY_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" +FIXTURE_REPOSITORY = "https://github.com/DataDog/ffe-system-test-data.git" +FIXTURE_REF_PATTERN = re.compile(r"[A-Za-z0-9._/-]+") + + +def _validate_fixture_ref(fixture_ref: str) -> None: + if ( + not fixture_ref + or fixture_ref.startswith("-") + or ".." in fixture_ref + or FIXTURE_REF_PATTERN.fullmatch(fixture_ref) is None + ): + msg = f"Invalid FFE fixture ref: {fixture_ref!r}" + raise ValueError(msg) + + +def _run_git(working_directory: Path, *arguments: str) -> str: + environment = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + } + result = subprocess.run( + ["git", *arguments], + cwd=working_directory, + env=environment, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _copy_entry(source: Path, destination: Path) -> None: + if source.is_symlink(): + msg = f"Refusing to copy symbolic link from FFE fixture repository: {source}" + raise ValueError(msg) + + if source.is_file(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return + + if not source.is_dir(): + msg = f"Refusing to copy unsupported filesystem entry from FFE fixture repository: {source}" + raise ValueError(msg) + + destination.mkdir(parents=True, exist_ok=True) + for child in sorted(source.iterdir()): + _copy_entry(child, destination / child.name) + + +def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: + # Import only the server evaluator corpus that this suite consumes. Other + # upstream suites (for example precomputed assignments) have different formats. + for name in ("ufc-config.json", "evaluation-cases"): + _copy_entry(source / name, snapshot / name) + + +def _validate_fixture_snapshot(snapshot: Path) -> int: + config_path = snapshot / "ufc-config.json" + cases_directory = snapshot / "evaluation-cases" + if not config_path.is_file() or not cases_directory.is_dir(): + msg = "FFE fixture repository does not contain the expected fixture layout" + raise ValueError(msg) + + with config_path.open(encoding="utf-8") as config_file: + config = json.load(config_file) + if not isinstance(config, dict) or not isinstance(config.get("flags"), dict) or not config["flags"]: + msg = "FFE UFC fixture must contain a non-empty flags object" + raise ValueError(msg) + + case_files = sorted(cases_directory.glob("*.json")) + if any(not path.is_file() or path.suffix != ".json" for path in cases_directory.iterdir()): + msg = "FFE evaluation-cases must contain only JSON files" + raise ValueError(msg) + if not case_files: + msg = "No FFE JSON fixture files found" + raise ValueError(msg) + + fixture_count = 0 + for case_file in case_files: + with case_file.open(encoding="utf-8") as fixture_file: + test_cases = json.load(fixture_file) + if not isinstance(test_cases, list): + msg = f"{case_file} must contain a JSON array of test cases" + raise TypeError(msg) + if not test_cases: + msg = f"{case_file} must contain at least one evaluation case" + raise ValueError(msg) + fixture_count += len(test_cases) + + if fixture_count == 0: + msg = "No FFE fixture test cases found" + raise ValueError(msg) + + return fixture_count + + +def _relative_files(directory: Path, excluded_files: frozenset[str] = frozenset()) -> list[Path]: + return sorted( + path.relative_to(directory) + for path in directory.rglob("*") + if path.is_file() and path.relative_to(directory).as_posix() not in excluded_files + ) + + +def _have_same_contents(snapshot: Path, destination: Path) -> bool: + if not destination.is_dir(): + return False + + snapshot_files = _relative_files(snapshot) + destination_files = _relative_files(destination, frozenset({"SOURCE.md"})) + if snapshot_files != destination_files: + return False + + return all( + (snapshot / relative_path).read_bytes() == (destination / relative_path).read_bytes() + for relative_path in snapshot_files + ) + + +def _write_source_metadata(snapshot: Path, source_commit: str) -> None: + (snapshot / "SOURCE.md").write_text( + f"""# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: {source_commit} + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. +""", + encoding="utf-8", + ) + + +def _set_github_outputs(source_commit: str, fixture_count: int, *, changed: bool) -> None: + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with Path(github_output).open("a", encoding="utf-8") as output_file: + output_file.write(f"source_commit={source_commit}\n") + output_file.write(f"fixture_count={fixture_count}\n") + output_file.write(f"changed={str(changed).lower()}\n") + + +def update_fixtures(fixture_ref: str) -> bool: + """Update the snapshot and return whether its allowed contents changed.""" + _validate_fixture_ref(fixture_ref) + + with tempfile.TemporaryDirectory(prefix="ffe-system-test-data-") as temporary_directory: + temporary_path = Path(temporary_directory) + source = temporary_path / "source" + snapshot = temporary_path / "snapshot" + source.mkdir() + snapshot.mkdir() + + _run_git(source, "init", "--quiet") + _run_git(source, "remote", "add", "origin", FIXTURE_REPOSITORY) + _run_git(source, "fetch", "--quiet", "--depth", "1", "origin", fixture_ref) + _run_git(source, "checkout", "--quiet", "--detach", "FETCH_HEAD") + source_commit = _run_git(source, "rev-parse", "HEAD") + + _copy_fixture_snapshot(source, snapshot) + fixture_count = _validate_fixture_snapshot(snapshot) + changed = not _have_same_contents(snapshot, FIXTURE_DESTINATION) + + if changed: + _write_source_metadata(snapshot, source_commit) + if FIXTURE_DESTINATION.exists(): + shutil.rmtree(FIXTURE_DESTINATION) + shutil.copytree(snapshot, FIXTURE_DESTINATION) + + _set_github_outputs(source_commit, fixture_count, changed=changed) + print(f"Checked FFE fixtures from DataDog/ffe-system-test-data@{source_commit}") + print(f"Loaded {fixture_count} canonical fixture cases") + print(f"Fixture snapshot changed: {str(changed).lower()}") + return changed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", default="main", help="Branch, tag, or commit from DataDog/ffe-system-test-data") + arguments = parser.parse_args() + update_fixtures(arguments.ref) + + +if __name__ == "__main__": + main() From 9e3ad6aae646be3e195754f06d546c24d1dbd119 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 21 Oct 2026 20:14:26 +0000 Subject: [PATCH 4/6] test(ffe): preserve evaluation details in parametric adapters Expose Node.js error codes and Ruby resolution details to canonical assertions. Convert Ruby numeric defaults to Float before calling the OpenFeature client. Environment: Datadog workspace --- .../build/docker/nodejs/parametric/server.js | 1 + utils/build/docker/ruby/parametric/server.rb | 24 ++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/utils/build/docker/nodejs/parametric/server.js b/utils/build/docker/nodejs/parametric/server.js index 78024c9231f..32b021b3e80 100644 --- a/utils/build/docker/nodejs/parametric/server.js +++ b/utils/build/docker/nodejs/parametric/server.js @@ -650,6 +650,7 @@ app.post('/ffe/evaluate', async (req, res) => { res.json({ value: details.value, reason: details.reason, + errorCode: details.errorCode, flagMetadata: details.flagMetadata ?? null }); }); diff --git a/utils/build/docker/ruby/parametric/server.rb b/utils/build/docker/ruby/parametric/server.rb index cf5fed16bb2..0461034fd50 100644 --- a/utils/build/docker/ruby/parametric/server.rb +++ b/utils/build/docker/ruby/parametric/server.rb @@ -869,20 +869,26 @@ def handle_ffe_evaluation(req, res) options = { flag_key: args.flag, default_value: args.default_value, evaluation_context: context } + options[:default_value] = args.default_value.to_f if args.variation_type == 'NUMERIC' - value = + details = case args.variation_type - when 'BOOLEAN'then client.fetch_boolean_value(**options) - when 'STRING' then client.fetch_string_value(**options) - when 'INTEGER' then client.fetch_integer_value(**options) - when 'NUMERIC' then client.fetch_float_value(**options) - when 'JSON' then client.fetch_object_value(**options) - else 'FATAL_UNEXPECTED_VARIATION_TYPE' + when 'BOOLEAN'then client.fetch_boolean_details(**options) + when 'STRING' then client.fetch_string_details(**options) + when 'INTEGER' then client.fetch_integer_details(**options) + when 'NUMERIC' then client.fetch_float_details(**options) + when 'JSON' then client.fetch_object_details(**options) + else raise ArgumentError, "Unexpected variation type: #{args.variation_type}" end - res.write({value: value, reason: 'DEFAULT'}.to_json) + res.write({ + value: details.value, + reason: details.reason, + errorCode: details.error_code, + flagMetadata: details.flag_metadata + }.to_json) rescue => e - res.write({value: args.default_value, reason: 'ERROR'}.to_json) + res.write({value: args.default_value, reason: 'ERROR', errorCode: 'GENERAL', errorMessage: e.message}.to_json) end end From 6ecd5f09ff35e435409c62d23496973e20dacd94 Mon Sep 17 00:00:00 2001 From: "tyler.potter" Date: Wed, 21 Oct 2026 21:30:22 +0000 Subject: [PATCH 5/6] Tighten canonical evaluation result checks Limit response assertions to API result fields, preserve successful error-code absence checks, and refresh dependent fixtures. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .../test_ffe/ffe-system-test-data/SOURCE.md | 2 +- .../test-case-dependent-flags.json | 24 ++++ .../ffe-system-test-data/ufc-config.json | 104 ++++++++++++++++++ .../test_ffe/test_dynamic_evaluation.py | 12 +- 4 files changed, 138 insertions(+), 4 deletions(-) diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index 3e3435f3b52..0a907783d5f 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -3,7 +3,7 @@ These files are copied from the canonical FFE fixture repository. Canonical source: https://github.com/DataDog/ffe-system-test-data -Source commit: 385413b4ad470a1f5bad7a4c4ec96a6aeb08489b +Source commit: cb4a8620adb90f4ada757e1042e346efa8035364 Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot. diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-dependent-flags.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-dependent-flags.json index 840044840b5..b91d49d41f6 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-dependent-flags.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-dependent-flags.json @@ -11,6 +11,30 @@ "targetingKey": "dependent-direct", "variationType": "STRING" }, + { + "description": "ONE_OF does not match when the prerequisite variant key is absent from the allowed list.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-one-of-miss-root", + "result": { + "reason": "STATIC", + "value": "one-of-fallback" + }, + "targetingKey": "dependent-one-of-miss", + "variationType": "STRING" + }, + { + "description": "NOT_ONE_OF does not match when the prerequisite variant key is present in the excluded list.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-not-one-of-hit-root", + "result": { + "reason": "STATIC", + "value": "not-one-of-fallback" + }, + "targetingKey": "dependent-not-one-of-hit", + "variationType": "STRING" + }, { "description": "A depth-two tree evaluates its leaf and intermediate prerequisite before matching the root.", "attributes": {}, diff --git a/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index c0089d0d245..ca100adf610 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -5170,6 +5170,110 @@ } ] }, + "dependent-one-of-miss-root": { + "key": "dependent-one-of-miss-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "one-of-fallback" + } + }, + "allocations": [ + { + "key": "variant-not-in-list", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "off" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-not-one-of-hit-root": { + "key": "dependent-not-one-of-hit-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "not-one-of-fallback" + } + }, + "allocations": [ + { + "key": "variant-in-excluded-list", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "NOT_ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, "dependent-transitive-parent": { "key": "dependent-transitive-parent", "enabled": true, diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 9f0de2647bf..ffb29c34060 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -203,11 +203,17 @@ def _assert_evaluation_cases( f"Test case {i} in {test_case_file} failed: FFE provider did not load RC data after " f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" ) - for field, expected_value in expected_result.items(): - assert result.get(field) == expected_value, ( + for field in ("value", "reason", "errorCode"): + if field not in expected_result: + continue + assert result.get(field) == expected_result[field], ( f"Test case {i} in {test_case_file} failed: " f"flag='{flag}', targetingKey='{targeting_key}', field='{field}', " - f"expected={expected_value!r}, actual={result.get(field)!r}" + f"expected={expected_result[field]!r}, actual={result.get(field)!r}" + ) + if "errorCode" not in expected_result and expected_result.get("reason") != "ERROR": + assert result.get("errorCode") in {None, ""}, ( + f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" ) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) From 0cebbe7939e2c5250128ba31dd4eb4c1131598b0 Mon Sep 17 00:00:00 2001 From: Tyler Potter Date: Thu, 22 Oct 2026 10:34:25 -0600 Subject: [PATCH 6/6] test(ffe): run dependent fixtures through canonical test Remove dependency-specific case partitioning and manifest exemptions so every imported evaluation case uses the shared dynamic-evaluation path. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- manifests/dotnet.yml | 1 - manifests/golang.yml | 1 - manifests/java.yml | 1 - manifests/nodejs.yml | 1 - manifests/php.yml | 1 - manifests/python.yml | 1 - manifests/ruby.yml | 1 - manifests/rust.yml | 1 - .../test_ffe/test_dynamic_evaluation.py | 51 +++---------------- 9 files changed, 8 insertions(+), 51 deletions(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 5fff0973bfc..7db6bf29e45 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1182,7 +1182,6 @@ manifest: - declaration: bug (FFL-3325) component_version: <3.55.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=3.54.0' # TODO: a lower version might be supported diff --git a/manifests/golang.yml b/manifests/golang.yml index e847cf70cd2..d498b6a39ab 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2166,7 +2166,6 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.10.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index 8f5bb656936..bd3559e9466 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4245,7 +4245,6 @@ manifest: : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_serial_id_metadata.py: v1.66.0 tests/parametric/test_ffe/test_span_enrichment.py: '>=1.64.2' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index e9cbbb458d1..94b6334410c 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2658,7 +2658,6 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: *ref_5_56_0 tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_serial_id_metadata.py: v6.18.0 tests/parametric/test_ffe/test_span_enrichment.py: # TODO: a lower version might be supported - declaration: missing_feature (dd-trace-js#8343) diff --git a/manifests/php.yml b/manifests/php.yml index af561a4ec4d..64d0e027b3d 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1618,7 +1618,6 @@ manifest: : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.23.3' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index a4d7a80da0d..22b9f37c023 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -3064,7 +3064,6 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=4.9.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 2e422cc45ed..28350f8caea 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2679,7 +2679,6 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_extract_invalid: # Easy win for all weblogs and version 2.27.0 diff --git a/manifests/rust.yml b/manifests/rust.yml index b5cefcd2728..67215ed811e 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -264,7 +264,6 @@ manifest: tests/parametric/test_extract_behavior.py: '>=0.4.1-dev' tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported - tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flag_evaluation: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index ffb29c34060..c9138cbe6d9 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -33,36 +33,17 @@ def _load_ufc_fixture() -> dict[str, Any]: return json.load(f) -def _contains_flag_evaluation(value: object) -> bool: - if isinstance(value, dict): - return "flagEvaluation" in value or any(_contains_flag_evaluation(child) for child in value.values()) - if isinstance(value, list): - return any(_contains_flag_evaluation(child) for child in value) - return False - - -def _get_test_case_files(ufc: dict[str, Any]) -> tuple[list[str], list[str]]: - """Partition canonical cases by evaluator capability without naming fixture files.""" +def _get_test_case_files() -> list[str]: + """Get all canonical evaluation case files.""" case_files = sorted(path.name for path in FFE_EVALUATION_CASES_DIR.glob("*.json")) if not case_files: raise FileNotFoundError(f"No canonical evaluation cases found in {FFE_EVALUATION_CASES_DIR}") - - ordinary: list[str] = [] - dependent: list[str] = [] - flags = ufc["flags"] - for case_file in case_files: - with (FFE_EVALUATION_CASES_DIR / case_file).open() as fixture_file: - cases = json.load(fixture_file) - destination = ( - dependent if any(_contains_flag_evaluation(flags.get(case["flag"], {})) for case in cases) else ordinary - ) - destination.append(case_file) - return ordinary, dependent + return case_files # Load fixture at module level for reuse across tests UFC_FIXTURE_DATA = _load_ufc_fixture() -ORDINARY_TEST_CASE_FILES, DEPENDENT_TEST_CASE_FILES = _get_test_case_files(UFC_FIXTURE_DATA) +ALL_TEST_CASE_FILES = _get_test_case_files() DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", @@ -161,12 +142,10 @@ def test_ffe_remote_config(self, test_agent: TestAgentAPI, test_library: APMLibr assert apply_state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value assert apply_state["product"] == RC_PRODUCT - def _assert_evaluation_cases( - self, - test_case_file: str, - test_agent: TestAgentAPI, - test_library: APMLibrary, - ) -> None: + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + @parametrize("test_case_file", ALL_TEST_CASE_FILES) + def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """Test every expected result field present in each canonical fixture case.""" # Load the test case file test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file @@ -215,17 +194,3 @@ def _assert_evaluation_cases( assert result.get("errorCode") in {None, ""}, ( f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" ) - - @parametrize("library_env", [{**DEFAULT_ENVVARS}]) - @parametrize("test_case_file", ORDINARY_TEST_CASE_FILES) - def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: - """Test every expected result field present in each canonical fixture case.""" - self._assert_evaluation_cases(test_case_file, test_agent, test_library) - - @parametrize("library_env", [{**DEFAULT_ENVVARS}]) - @parametrize("test_case_file", DEPENDENT_TEST_CASE_FILES) - def test_ffe_dependent_flag_evaluation( - self, test_case_file: str, test_agent: TestAgentAPI, test_library: APMLibrary - ) -> None: - """Run canonical cases whose root flag requires dependent evaluation support.""" - self._assert_evaluation_cases(test_case_file, test_agent, test_library)