From 6c8a9500ff364c1059e8264f42dfea7e5a690fb2 Mon Sep 17 00:00:00 2001 From: "tyler.potter" Date: Fri, 30 Oct 2026 20:03:03 +0000 Subject: [PATCH 1/4] test(ffe): import canonical feature suites Extend the unified fixture snapshot to include rules-based feature suites and run dependent flags through the shared evaluator harness without maintaining copied fixtures. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .../cases/dependent-flags.json | 112 ------- .../test_ffe/ffe-system-test-data/SOURCE.md | 2 +- .../cases/dependent-flags.json | 316 ++++++++++++++++++ .../dependent-flags/ufc-config.json | 202 +++++++++++ .../test_ffe/test_dynamic_evaluation.py | 150 +++------ .../test_the_test/test_update_ffe_fixtures.py | 34 +- utils/scripts/update_ffe_fixtures.py | 40 ++- 7 files changed, 632 insertions(+), 224 deletions(-) delete mode 100644 tests/parametric/test_ffe/dependent-flags/cases/dependent-flags.json create mode 100644 tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json rename tests/parametric/test_ffe/{ => ffe-system-test-data/evaluation-suites}/dependent-flags/ufc-config.json (73%) diff --git a/tests/parametric/test_ffe/dependent-flags/cases/dependent-flags.json b/tests/parametric/test_ffe/dependent-flags/cases/dependent-flags.json deleted file mode 100644 index b91d49d41f6..00000000000 --- a/tests/parametric/test_ffe/dependent-flags/cases/dependent-flags.json +++ /dev/null @@ -1,112 +0,0 @@ -[ - { - "description": "A root flag matches a direct prerequisite by variant key.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-direct-root", - "result": { - "reason": "TARGETING_MATCH", - "value": "direct-match" - }, - "targetingKey": "dependent-direct", - "variationType": "STRING" - }, - { - "description": "ONE_OF does not match when the prerequisite variant key is absent from the allowed list.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-one-of-miss-root", - "result": { - "reason": "STATIC", - "value": "one-of-fallback" - }, - "targetingKey": "dependent-one-of-miss", - "variationType": "STRING" - }, - { - "description": "NOT_ONE_OF does not match when the prerequisite variant key is present in the excluded list.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-not-one-of-hit-root", - "result": { - "reason": "STATIC", - "value": "not-one-of-fallback" - }, - "targetingKey": "dependent-not-one-of-hit", - "variationType": "STRING" - }, - { - "description": "A depth-two tree evaluates its leaf and intermediate prerequisite before matching the root.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-transitive-root", - "result": { - "reason": "TARGETING_MATCH", - "value": "transitive-match" - }, - "targetingKey": "dependent-transitive", - "variationType": "STRING" - }, - { - "description": "A false attribute condition short-circuits before an otherwise fatal missing prerequisite and selects the root fallback.", - "attributes": { - "gate": "closed" - }, - "defaultValue": "caller-default", - "flag": "dependent-short-circuit-root", - "result": { - "reason": "STATIC", - "value": "short-circuit-fallback" - }, - "targetingKey": "dependent-short-circuit", - "variationType": "STRING" - }, - { - "description": "Two membership conditions against the same prerequisite evaluate successfully.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-memoized-root", - "result": { - "reason": "TARGETING_MATCH", - "value": "memoized-match" - }, - "targetingKey": "dependent-memoized", - "variationType": "STRING" - }, - { - "description": "A successful prerequisite followed by a missing prerequisite aborts the graph and returns the caller default.", - "attributes": {}, - "defaultValue": "atomic-default", - "flag": "dependent-atomic-error-root", - "result": { - "reason": "ERROR", - "value": "atomic-default" - }, - "targetingKey": "dependent-atomic-error", - "variationType": "STRING" - }, - { - "description": "With maxDependencyDepth 2, a third dependency edge aborts the graph and returns the caller default.", - "attributes": {}, - "defaultValue": "depth-default", - "flag": "dependent-depth-three-root", - "result": { - "reason": "ERROR", - "value": "depth-default" - }, - "targetingKey": "dependent-over-depth", - "variationType": "STRING" - }, - { - "description": "A dependency cycle aborts the graph and returns the caller default.", - "attributes": {}, - "defaultValue": "cycle-default", - "flag": "dependent-cycle-a", - "result": { - "reason": "ERROR", - "value": "cycle-default" - }, - "targetingKey": "dependent-cycle", - "variationType": "STRING" - } -] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index 50d004210d7..15f60538d05 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -3,7 +3,7 @@ These files are copied from the canonical FFE fixture repository. Canonical source: https://github.com/DataDog/ffe-system-test-data -Source commit: caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466 +Source commit: d8911fd2a7557662944745849e4d9de554262e8d Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot. diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json new file mode 100644 index 00000000000..76cde16936f --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json @@ -0,0 +1,316 @@ +[ + { + "description": "A root flag matches a direct prerequisite by variant key.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-direct-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "direct-match" + }, + "targetingKey": "dependent-direct", + "variationType": "STRING" + }, + { + "description": "ONE_OF does not match when the prerequisite variant key is absent from the allowed list. The prerequisite retains its evaluation event but not its exposure because the containing allocation is rejected.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-one-of-miss-root", + "result": { + "reason": "STATIC", + "value": "one-of-fallback" + }, + "targetingKey": "dependent-one-of-miss", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-one-of-miss-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-one-of-miss-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "NOT_ONE_OF does not match when the prerequisite variant key is present in the excluded list. The prerequisite retains its evaluation event but not its exposure because the containing allocation is rejected.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-not-one-of-hit-root", + "result": { + "reason": "STATIC", + "value": "not-one-of-fallback" + }, + "targetingKey": "dependent-not-one-of-hit", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-not-one-of-hit-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-not-one-of-hit-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "A depth-two tree evaluates its leaf and intermediate prerequisite before matching the root.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-transitive-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "transitive-match" + }, + "targetingKey": "dependent-transitive", + "variationType": "STRING" + }, + { + "description": "A false attribute condition short-circuits before an otherwise fatal missing prerequisite and selects the root fallback.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-short-circuit-root", + "result": { + "reason": "STATIC", + "value": "short-circuit-fallback" + }, + "targetingKey": "dependent-short-circuit", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-short-circuit-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-short-circuit-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "A prerequisite evaluated before a later false condition retains its evaluation event but not its exposure because the containing targeting rule is not selected.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-inapplicable-rule-root", + "result": { + "reason": "STATIC", + "value": "inapplicable-rule-fallback" + }, + "targetingKey": "dependent-inapplicable-rule", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-inapplicable-rule-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-inapplicable-rule-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "When every condition matches, the selected rule retains both the prerequisite and root exposures.", + "attributes": { + "gate": "open" + }, + "defaultValue": "caller-default", + "flag": "dependent-inapplicable-rule-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "inapplicable-rule-match" + }, + "targetingKey": "dependent-applicable-rule", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "flag": "dependent-leaf-on" + }, + { + "allocation": "prerequisite-before-false-condition", + "flag": "dependent-inapplicable-rule-root", + "variant": "matched" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-inapplicable-rule-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "A prerequisite first used by a rejected rule remains eligible for one exposure when a later selected rule reuses it.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-reuse-after-rejection-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "reuse-after-rejection-match" + }, + "targetingKey": "dependent-reuse-after-rejection", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "_count": 1, + "flag": "dependent-leaf-on" + }, + { + "allocation": "selected-reuse", + "flag": "dependent-reuse-after-rejection-root", + "variant": "matched" + } + ], + "evaluationEvents": [ + { + "_count": 1, + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-reuse-after-rejection-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "When a later condition rejects a rule after a transitive dependency succeeds, neither the parent nor leaf exposure remains eligible.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-transitive-inapplicable-rule-root", + "result": { + "reason": "STATIC", + "value": "transitive-inapplicable-fallback" + }, + "targetingKey": "dependent-transitive-inapplicable-rule", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-transitive-inapplicable-rule-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-transitive-parent", + "errorCode": null + }, + { + "flag": "dependent-transitive-inapplicable-rule-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "Two membership conditions against the same prerequisite evaluate successfully.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-memoized-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "memoized-match" + }, + "targetingKey": "dependent-memoized", + "variationType": "STRING" + }, + { + "description": "A successful prerequisite followed by a missing prerequisite aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "atomic-default", + "flag": "dependent-atomic-error-root", + "result": { + "reason": "ERROR", + "value": "atomic-default" + }, + "targetingKey": "dependent-atomic-error", + "variationType": "STRING" + }, + { + "description": "With maxDependencyDepth 2, a third dependency edge aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "depth-default", + "flag": "dependent-depth-three-root", + "result": { + "reason": "ERROR", + "value": "depth-default" + }, + "targetingKey": "dependent-over-depth", + "variationType": "STRING" + }, + { + "description": "A dependency cycle aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "cycle-default", + "flag": "dependent-cycle-a", + "result": { + "reason": "ERROR", + "value": "cycle-default" + }, + "targetingKey": "dependent-cycle", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/dependent-flags/ufc-config.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json similarity index 73% rename from tests/parametric/test_ffe/dependent-flags/ufc-config.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json index d45b41c8c6b..e4fea47b37e 100644 --- a/tests/parametric/test_ffe/dependent-flags/ufc-config.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json @@ -427,6 +427,208 @@ } ] }, + "dependent-inapplicable-rule-root": { + "key": "dependent-inapplicable-rule-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "inapplicable-rule-match" + }, + "fallback": { + "key": "fallback", + "value": "inapplicable-rule-fallback" + } + }, + "allocations": [ + { + "key": "prerequisite-before-false-condition", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-reuse-after-rejection-root": { + "key": "dependent-reuse-after-rejection-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "reuse-after-rejection-match" + }, + "fallback": { + "key": "fallback", + "value": "reuse-after-rejection-fallback" + } + }, + "allocations": [ + { + "key": "rejected-first-use", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "selected-reuse", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-transitive-inapplicable-rule-root": { + "key": "dependent-transitive-inapplicable-rule-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "transitive-inapplicable-fallback" + } + }, + "allocations": [ + { + "key": "transitive-prerequisite-before-false-condition", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-transitive-parent" + }, + "operator": "ONE_OF", + "value": [ + "parent-on" + ] + }, + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, "dependent-memoized-root": { "key": "dependent-memoized-root", "enabled": true, diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 48088117087..904cd76a7cd 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -23,7 +23,7 @@ FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 FFE_SYSTEM_TEST_DATA_DIR = Path(__file__).parent / "ffe-system-test-data" FFE_EVALUATION_CASES_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-cases" -DEPENDENT_FLAGS_DIR = Path(__file__).parent / "dependent-flags" +DEPENDENT_FLAGS_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-suites" / "dependent-flags" DEPENDENT_FLAGS_CASES_DIR = DEPENDENT_FLAGS_DIR / "cases" parametrize = pytest.mark.parametrize @@ -158,6 +158,54 @@ def _ffe_evaluate_with_rc_retry( return result +def _assert_evaluation_cases( + test_case_file: Path, + ufc_data: dict[str, Any], + test_agent: TestAgentAPI, + test_library: APMLibrary, +) -> None: + with test_case_file.open() as fixture_file: + test_cases = json.load(fixture_file) + assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file.name}" + assert test_cases, f"No evaluation cases in {test_case_file.name}" + + platform = context.library.name + test_cases = _filter_test_cases_for_platform(test_cases, platform) + if not test_cases: + pytest.skip(f"No cases in {test_case_file.name} apply to platform {platform}") + + _set_and_wait_ffe_rc(test_agent, ufc_data) + assert test_library.ffe_start(ufc_data), "Failed to start FFE provider" + + for index, test_case in enumerate(test_cases): + expected_result = test_case["result"] + result = _ffe_evaluate_with_rc_retry( + test_library, + flag=test_case["flag"], + variation_type=test_case["variationType"], + default_value=test_case["defaultValue"], + targeting_key=test_case["targetingKey"], + attributes=test_case.get("attributes", {}), + ) + assert not _is_ffe_waiting_for_rc(result), ( + f"Test case {index} in {test_case_file.name} failed: FFE provider did not load RC data after " + f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" + ) + for field in ("value", "reason", "errorCode"): + if field not in expected_result: + continue + assert result.get(field) == expected_result[field], ( + f"Test case {index} in {test_case_file.name} failed: " + f"flag='{test_case['flag']}', targetingKey='{test_case['targetingKey']}', field='{field}', " + f"expected={expected_result[field]!r}, actual={result.get(field)!r}" + ) + if "errorCode" not in expected_result and expected_result.get("reason") != "ERROR": + assert result.get("errorCode") in {None, ""}, ( + f"Test case {index} in {test_case_file.name} failed: " + f"flag='{test_case['flag']}' unexpectedly returned errorCode={result.get('errorCode')!r}" + ) + + @scenarios.parametric @features.feature_flags_dynamic_evaluation class Test_Feature_Flag_Dynamic_Evaluation: @@ -188,66 +236,8 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI 4. Handles user targeting, attribute matching, and rollout percentages """ - # Load the test case file test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file - - with test_case_path.open() as f: - test_cases = json.load(f) - assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" - assert test_cases, f"No evaluation cases in {test_case_file}" - - platform = context.library.name - test_cases = _filter_test_cases_for_platform(test_cases, platform) - if not test_cases: - pytest.skip(f"No cases in {test_case_file} apply to platform {platform}") - - # Set up UFC Remote Config and wait for it to be applied - _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) - - # Initialize FFE provider - success = test_library.ffe_start(UFC_FIXTURE_DATA) - assert success, "Failed to start FFE provider" - - # Run each test case - for i, test_case in enumerate(test_cases): - flag = test_case["flag"] - variation_type = test_case["variationType"] - default_value = test_case["defaultValue"] - targeting_key = test_case["targetingKey"] - attributes = test_case.get("attributes", {}) - expected_result = test_case["result"] - - result = _ffe_evaluate_with_rc_retry( - test_library, - flag=flag, - variation_type=variation_type, - default_value=default_value, - targeting_key=targeting_key, - attributes=attributes, - ) - assert not _is_ffe_waiting_for_rc(result), ( - f"Test case {i} in {test_case_file} failed: FFE provider did not load RC data after " - f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" - ) - actual_value = result.get("value") - - # Assert the evaluation result matches expected value - assert actual_value == expected_result["value"], ( - f"Test case {i} in {test_case_file} failed: " - f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={result}" - ) - assert result.get("reason") == expected_result["reason"], ( - f"Test case {i} in {test_case_file}: expected reason={expected_result['reason']}, result={result}" - ) - if "errorCode" in expected_result: - assert result.get("errorCode") == expected_result["errorCode"], ( - f"Test case {i} in {test_case_file}: expected errorCode={expected_result['errorCode']}, result={result}" - ) - elif expected_result["reason"] != "ERROR": - assert result.get("errorCode") in {None, ""}, ( - f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" - ) + _assert_evaluation_cases(test_case_path, UFC_FIXTURE_DATA, test_agent, test_library) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) @parametrize("test_case_file", DEPENDENT_FLAGS_TEST_CASE_FILES) @@ -257,44 +247,8 @@ def test_ffe_dependent_flags( test_agent: TestAgentAPI, test_library: APMLibrary, ) -> None: - """Test dependent flags using only the fixtures in the dependent-flags suite.""" - with test_case_file.open() as f: - test_cases = json.load(f) - - _set_and_wait_ffe_rc(test_agent, DEPENDENT_FLAGS_UFC_DATA) - - success = test_library.ffe_start(DEPENDENT_FLAGS_UFC_DATA) - assert success, "Failed to start FFE provider" - - for i, test_case in enumerate(test_cases): - flag = test_case["flag"] - targeting_key = test_case["targetingKey"] - expected_result = test_case["result"] - result = _ffe_evaluate_with_rc_retry( - test_library, - flag=flag, - variation_type=test_case["variationType"], - default_value=test_case["defaultValue"], - targeting_key=targeting_key, - attributes=test_case.get("attributes", {}), - ) - assert not _is_ffe_waiting_for_rc(result), ( - f"Test case {i} in {test_case_file.name} failed: FFE provider did not load RC data after " - f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" - ) - for field in ("value", "reason", "errorCode"): - if field not in expected_result: - continue - assert result.get(field) == expected_result[field], ( - f"Test case {i} in {test_case_file.name} failed: " - f"flag='{flag}', targetingKey='{targeting_key}', field='{field}', " - f"expected={expected_result[field]!r}, actual={result.get(field)!r}" - ) - if "errorCode" not in expected_result and expected_result.get("reason") != "ERROR": - assert result.get("errorCode") in {None, ""}, ( - f"Test case {i} in {test_case_file.name} failed: " - f"flag='{flag}' unexpectedly returned errorCode={result.get('errorCode')!r}" - ) + """Run the imported dependent-flags feature suite through the shared evaluator harness.""" + _assert_evaluation_cases(test_case_file, DEPENDENT_FLAGS_UFC_DATA, test_agent, test_library) @scenarios.parametric diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py index 8f53acd82ee..fc779fa1eec 100644 --- a/tests/test_the_test/test_update_ffe_fixtures.py +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -22,14 +22,22 @@ def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: update_ffe_fixtures._validate_fixture_ref(fixture_ref) +def _write_feature_suite(source: Path, cases: str = '[{"flag": "dependent"}]') -> None: + suite = source / "evaluation-suites" / "dependent-flags" + (suite / "cases").mkdir(parents=True) + (suite / "ufc-config.json").write_text('{"flags": {"dependent": {}}}', encoding="utf-8") + (suite / "cases" / "dependent.json").write_text(cases, encoding="utf-8") + + @scenarios.test_the_test -def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path) -> None: +def test_copy_fixture_snapshot_imports_only_rules_based_evaluator_data(tmp_path: Path) -> None: source = tmp_path / "source" snapshot = tmp_path / "snapshot" (source / "evaluation-cases").mkdir(parents=True) snapshot.mkdir() (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") (source / "ufc-config.json").write_text("{}", encoding="utf-8") + _write_feature_suite(source) (source / "README.md").write_text("upstream documentation", encoding="utf-8") (source / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") (source / "precomputed-assignments").mkdir() @@ -38,6 +46,8 @@ def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path assert (snapshot / "ufc-config.json").is_file() assert (snapshot / "evaluation-cases" / "case.json").is_file() + assert (snapshot / "evaluation-suites" / "dependent-flags" / "ufc-config.json").is_file() + assert (snapshot / "evaluation-suites" / "dependent-flags" / "cases" / "dependent.json").is_file() assert not (snapshot / "README.md").exists() assert not (snapshot / "AGENTS.md").exists() assert not (snapshot / "precomputed-assignments").exists() @@ -51,6 +61,7 @@ def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: snapshot.mkdir() (source / "evaluation-cases").mkdir() (source / "ufc-config.json").write_text("{}", encoding="utf-8") + _write_feature_suite(source) target = source / "target.json" target.write_text("{}", encoding="utf-8") (source / "evaluation-cases" / "linked.json").symlink_to(target) @@ -66,8 +77,22 @@ def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") (cases_directory / "first.json").write_text('[{"flag": "one"}]', encoding="utf-8") (cases_directory / "second.json").write_text('[{"flag": "two"}, {"flag": "three"}]', encoding="utf-8") + _write_feature_suite(tmp_path) + + assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 4 + + +@scenarios.test_the_test +def test_validate_fixture_snapshot_rejects_unexpected_feature_suite_entries(tmp_path: Path) -> None: + cases_directory = tmp_path / "evaluation-cases" + cases_directory.mkdir() + (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") + (cases_directory / "case.json").write_text('[{"flag": "one"}]', encoding="utf-8") + _write_feature_suite(tmp_path) + (tmp_path / "evaluation-suites" / "dependent-flags" / "README.md").write_text("unexpected", encoding="utf-8") - assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 + with pytest.raises(ValueError, match="must contain only ufc-config.json and cases"): + update_ffe_fixtures._validate_fixture_snapshot(tmp_path) @scenarios.test_the_test @@ -95,6 +120,7 @@ def test_update_fixtures_preserves_snapshot_on_validation_failure( (source / "evaluation-cases").mkdir(parents=True) (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + _write_feature_suite(source) destination.mkdir() (destination / "SOURCE.md").write_text("original snapshot", encoding="utf-8") @@ -121,6 +147,7 @@ def test_update_fixtures_records_source_and_removes_stale_files( (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") case_file = source / "evaluation-cases" / "case.json" case_file.write_text('[{"flag": "flag"}]', encoding="utf-8") + _write_feature_suite(source) destination.mkdir() (destination / "stale.json").write_text("[]", encoding="utf-8") @@ -135,9 +162,10 @@ def fake_git(working_directory: Path, *arguments: str) -> str: assert update_ffe_fixtures.update_fixtures("main") assert not (destination / "stale.json").exists() assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() + assert (destination / "evaluation-suites" / "dependent-flags" / "cases" / "dependent.json").is_file() metadata = (destination / "SOURCE.md").read_bytes() assert ("a" * 40).encode() in metadata - assert "fixture_count=1\nchanged=true\n" in output.read_text() + assert "fixture_count=2\nchanged=true\n" in output.read_text() (source / "README.md").write_text("documentation-only upstream change", encoding="utf-8") assert not update_ffe_fixtures.update_fixtures("main") diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py index e7f3b95f484..224fa5023a0 100644 --- a/utils/scripts/update_ffe_fixtures.py +++ b/utils/scripts/update_ffe_fixtures.py @@ -64,15 +64,13 @@ def _copy_entry(source: Path, destination: Path) -> None: def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: - # Import only the server evaluator corpus that this suite consumes. Other - # upstream suites (for example precomputed assignments) have different formats. - for name in ("ufc-config.json", "evaluation-cases"): + # Import only rules-based evaluator corpora. Precomputed assignments use a + # different format and are consumed by a different test harness. + for name in ("ufc-config.json", "evaluation-cases", "evaluation-suites"): _copy_entry(source / name, snapshot / name) -def _validate_fixture_snapshot(snapshot: Path) -> int: - config_path = snapshot / "ufc-config.json" - cases_directory = snapshot / "evaluation-cases" +def _validate_evaluation_suite(config_path: Path, cases_directory: Path) -> int: if not config_path.is_file() or not cases_directory.is_dir(): msg = "FFE fixture repository does not contain the expected fixture layout" raise ValueError(msg) @@ -91,7 +89,7 @@ def _validate_fixture_snapshot(snapshot: Path) -> int: msg = "No FFE JSON fixture files found" raise ValueError(msg) - fixture_count = 0 + case_count = 0 for case_file in case_files: with case_file.open(encoding="utf-8") as fixture_file: test_cases = json.load(fixture_file) @@ -101,11 +99,33 @@ def _validate_fixture_snapshot(snapshot: Path) -> int: if not test_cases: msg = f"{case_file} must contain at least one evaluation case" raise ValueError(msg) - fixture_count += len(test_cases) + case_count += len(test_cases) + + return case_count + + +def _validate_fixture_snapshot(snapshot: Path) -> int: + fixture_count = _validate_evaluation_suite(snapshot / "ufc-config.json", snapshot / "evaluation-cases") - if fixture_count == 0: - msg = "No FFE fixture test cases found" + suites_directory = snapshot / "evaluation-suites" + if not suites_directory.is_dir(): + msg = "FFE fixture repository does not contain evaluation-suites" + raise ValueError(msg) + suite_directories = sorted(suites_directory.iterdir()) + if not suite_directories: + msg = "FFE evaluation-suites must contain at least one feature suite" raise ValueError(msg) + for suite_directory in suite_directories: + if not suite_directory.is_dir() or suite_directory.is_symlink(): + msg = "FFE evaluation-suites must contain only suite directories" + raise ValueError(msg) + if {entry.name for entry in suite_directory.iterdir()} != {"ufc-config.json", "cases"}: + msg = f"FFE feature suite {suite_directory.name} must contain only ufc-config.json and cases" + raise ValueError(msg) + fixture_count += _validate_evaluation_suite( + suite_directory / "ufc-config.json", + suite_directory / "cases", + ) return fixture_count From ae68dc1ebff63068bd4975b58116fc180e1d3f84 Mon Sep 17 00:00:00 2001 From: Tyler Potter Date: Mon, 2 Nov 2026 09:52:19 -0700 Subject: [PATCH 2/4] test(ffe): refresh canonical feature suites Import the latest dependent-flags corpus, preserve required result assertions, and keep importer unit tests out of this harness change. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- .../test_ffe/ffe-system-test-data/SOURCE.md | 2 +- .../cases/dependent-flags.json | 27 ++++++++ .../dependent-flags/ufc-config.json | 64 +++++++++++++++++++ .../test_ffe/test_dynamic_evaluation.py | 12 ++-- .../test_the_test/test_update_ffe_fixtures.py | 34 +--------- 5 files changed, 103 insertions(+), 36 deletions(-) diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index 15f60538d05..ec63232bc34 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -3,7 +3,7 @@ These files are copied from the canonical FFE fixture repository. Canonical source: https://github.com/DataDog/ffe-system-test-data -Source commit: d8911fd2a7557662944745849e4d9de554262e8d +Source commit: c8cc8da239341748dc8c30c2145c0f0b5432c958 Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot. diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json index 76cde16936f..44e92da23f5 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json @@ -277,6 +277,33 @@ "targetingKey": "dependent-memoized", "variationType": "STRING" }, + { + "description": "A reached prerequisite that produces no assignment aborts the graph before a later unconditional root allocation can return or publish an exposure.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-no-assignment-root", + "result": { + "reason": "ERROR", + "errorCode": "GENERAL", + "value": "caller-default" + }, + "targetingKey": "dependent-no-assignment", + "variationType": "STRING", + "expectations": { + "exposures": [], + "evaluationEvents": [ + { + "flag": "dependent-no-assignment-parent", + "errorCode": "GENERAL" + }, + { + "flag": "dependent-no-assignment-root", + "errorCode": "GENERAL" + } + ], + "noUnmatchedEvents": true + } + }, { "description": "A successful prerequisite followed by a missing prerequisite aborts the graph and returns the caller default.", "attributes": {}, diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json index e4fea47b37e..dd15f9346cc 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json @@ -676,6 +676,70 @@ } ] }, + "dependent-no-assignment-parent": { + "key": "dependent-no-assignment-parent", + "enabled": true, + "variationType": "STRING", + "variations": { + "assigned": { + "key": "assigned", + "value": "must-never-be-returned-no-assignment-parent" + } + }, + "allocations": [] + }, + "dependent-no-assignment-root": { + "key": "dependent-no-assignment-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned-no-assignment-match" + }, + "fallback": { + "key": "fallback", + "value": "must-not-be-returned-no-assignment-fallback" + } + }, + "allocations": [ + { + "key": "parent-assigned", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-no-assignment-parent" + }, + "operator": "ONE_OF", + "value": [ + "assigned" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "must-not-be-reached-fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, "dependent-atomic-error-root": { "key": "dependent-atomic-error-root", "enabled": true, diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 904cd76a7cd..208a5721c90 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -191,15 +191,19 @@ def _assert_evaluation_cases( f"Test case {index} in {test_case_file.name} failed: FFE provider did not load RC data after " f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" ) - for field in ("value", "reason", "errorCode"): - if field not in expected_result: - continue + for field in ("value", "reason"): assert result.get(field) == expected_result[field], ( f"Test case {index} in {test_case_file.name} failed: " f"flag='{test_case['flag']}', targetingKey='{test_case['targetingKey']}', field='{field}', " f"expected={expected_result[field]!r}, actual={result.get(field)!r}" ) - if "errorCode" not in expected_result and expected_result.get("reason") != "ERROR": + if "errorCode" in expected_result: + assert result.get("errorCode") == expected_result["errorCode"], ( + f"Test case {index} in {test_case_file.name} failed: " + f"flag='{test_case['flag']}', targetingKey='{test_case['targetingKey']}', field='errorCode', " + f"expected={expected_result['errorCode']!r}, actual={result.get('errorCode')!r}" + ) + elif expected_result["reason"] != "ERROR": assert result.get("errorCode") in {None, ""}, ( f"Test case {index} in {test_case_file.name} failed: " f"flag='{test_case['flag']}' unexpectedly returned errorCode={result.get('errorCode')!r}" diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py index fc779fa1eec..8f53acd82ee 100644 --- a/tests/test_the_test/test_update_ffe_fixtures.py +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -22,22 +22,14 @@ def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: update_ffe_fixtures._validate_fixture_ref(fixture_ref) -def _write_feature_suite(source: Path, cases: str = '[{"flag": "dependent"}]') -> None: - suite = source / "evaluation-suites" / "dependent-flags" - (suite / "cases").mkdir(parents=True) - (suite / "ufc-config.json").write_text('{"flags": {"dependent": {}}}', encoding="utf-8") - (suite / "cases" / "dependent.json").write_text(cases, encoding="utf-8") - - @scenarios.test_the_test -def test_copy_fixture_snapshot_imports_only_rules_based_evaluator_data(tmp_path: Path) -> None: +def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path) -> None: source = tmp_path / "source" snapshot = tmp_path / "snapshot" (source / "evaluation-cases").mkdir(parents=True) snapshot.mkdir() (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") (source / "ufc-config.json").write_text("{}", encoding="utf-8") - _write_feature_suite(source) (source / "README.md").write_text("upstream documentation", encoding="utf-8") (source / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") (source / "precomputed-assignments").mkdir() @@ -46,8 +38,6 @@ def test_copy_fixture_snapshot_imports_only_rules_based_evaluator_data(tmp_path: assert (snapshot / "ufc-config.json").is_file() assert (snapshot / "evaluation-cases" / "case.json").is_file() - assert (snapshot / "evaluation-suites" / "dependent-flags" / "ufc-config.json").is_file() - assert (snapshot / "evaluation-suites" / "dependent-flags" / "cases" / "dependent.json").is_file() assert not (snapshot / "README.md").exists() assert not (snapshot / "AGENTS.md").exists() assert not (snapshot / "precomputed-assignments").exists() @@ -61,7 +51,6 @@ def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: snapshot.mkdir() (source / "evaluation-cases").mkdir() (source / "ufc-config.json").write_text("{}", encoding="utf-8") - _write_feature_suite(source) target = source / "target.json" target.write_text("{}", encoding="utf-8") (source / "evaluation-cases" / "linked.json").symlink_to(target) @@ -77,22 +66,8 @@ def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") (cases_directory / "first.json").write_text('[{"flag": "one"}]', encoding="utf-8") (cases_directory / "second.json").write_text('[{"flag": "two"}, {"flag": "three"}]', encoding="utf-8") - _write_feature_suite(tmp_path) - - assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 4 - - -@scenarios.test_the_test -def test_validate_fixture_snapshot_rejects_unexpected_feature_suite_entries(tmp_path: Path) -> None: - cases_directory = tmp_path / "evaluation-cases" - cases_directory.mkdir() - (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") - (cases_directory / "case.json").write_text('[{"flag": "one"}]', encoding="utf-8") - _write_feature_suite(tmp_path) - (tmp_path / "evaluation-suites" / "dependent-flags" / "README.md").write_text("unexpected", encoding="utf-8") - with pytest.raises(ValueError, match="must contain only ufc-config.json and cases"): - update_ffe_fixtures._validate_fixture_snapshot(tmp_path) + assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 @scenarios.test_the_test @@ -120,7 +95,6 @@ def test_update_fixtures_preserves_snapshot_on_validation_failure( (source / "evaluation-cases").mkdir(parents=True) (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") - _write_feature_suite(source) destination.mkdir() (destination / "SOURCE.md").write_text("original snapshot", encoding="utf-8") @@ -147,7 +121,6 @@ def test_update_fixtures_records_source_and_removes_stale_files( (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") case_file = source / "evaluation-cases" / "case.json" case_file.write_text('[{"flag": "flag"}]', encoding="utf-8") - _write_feature_suite(source) destination.mkdir() (destination / "stale.json").write_text("[]", encoding="utf-8") @@ -162,10 +135,9 @@ def fake_git(working_directory: Path, *arguments: str) -> str: assert update_ffe_fixtures.update_fixtures("main") assert not (destination / "stale.json").exists() assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() - assert (destination / "evaluation-suites" / "dependent-flags" / "cases" / "dependent.json").is_file() metadata = (destination / "SOURCE.md").read_bytes() assert ("a" * 40).encode() in metadata - assert "fixture_count=2\nchanged=true\n" in output.read_text() + assert "fixture_count=1\nchanged=true\n" in output.read_text() (source / "README.md").write_text("documentation-only upstream change", encoding="utf-8") assert not update_ffe_fixtures.update_fixtures("main") From 48baa6d15a70668e941a93ba49c72d325027cc42 Mon Sep 17 00:00:00 2001 From: Tyler Potter Date: Mon, 2 Nov 2026 10:06:37 -0700 Subject: [PATCH 3/4] fix(ffe): accept corpora without feature suites Keep fixture imports compatible with rules-based corpora that do not yet define optional feature suites. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- utils/scripts/update_ffe_fixtures.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py index 224fa5023a0..9b9fbf2289e 100644 --- a/utils/scripts/update_ffe_fixtures.py +++ b/utils/scripts/update_ffe_fixtures.py @@ -66,9 +66,13 @@ def _copy_entry(source: Path, destination: Path) -> None: def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: # Import only rules-based evaluator corpora. Precomputed assignments use a # different format and are consumed by a different test harness. - for name in ("ufc-config.json", "evaluation-cases", "evaluation-suites"): + for name in ("ufc-config.json", "evaluation-cases"): _copy_entry(source / name, snapshot / name) + feature_suites = source / "evaluation-suites" + if feature_suites.exists() or feature_suites.is_symlink(): + _copy_entry(feature_suites, snapshot / feature_suites.name) + def _validate_evaluation_suite(config_path: Path, cases_directory: Path) -> int: if not config_path.is_file() or not cases_directory.is_dir(): @@ -108,8 +112,10 @@ def _validate_fixture_snapshot(snapshot: Path) -> int: fixture_count = _validate_evaluation_suite(snapshot / "ufc-config.json", snapshot / "evaluation-cases") suites_directory = snapshot / "evaluation-suites" - if not suites_directory.is_dir(): - msg = "FFE fixture repository does not contain evaluation-suites" + if not suites_directory.exists(): + return fixture_count + if not suites_directory.is_dir() or suites_directory.is_symlink(): + msg = "FFE evaluation-suites must be a directory" raise ValueError(msg) suite_directories = sorted(suites_directory.iterdir()) if not suite_directories: From 3780629e61c8b85f0866e14ca9b3e79fc6230810 Mon Sep 17 00:00:00 2001 From: Tyler Potter Date: Mon, 2 Nov 2026 12:01:27 -0600 Subject: [PATCH 4/4] fix(ffe): tolerate absent feature suite Keep core dynamic evaluation tests collectable when the optional dependent-flags corpus is unavailable. Environment: Datadog workspace Co-Authored-By: Codex GPT-5 --- tests/parametric/test_ffe/test_dynamic_evaluation.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 208a5721c90..b35d3280840 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -62,9 +62,12 @@ def _filter_test_cases_for_platform(test_cases: list[dict[str, Any]], platform: return applicable_cases -def _get_dependent_flags_test_case_files() -> list[Path]: - """Get the test case files owned by the dependent-flags suite.""" - return sorted(DEPENDENT_FLAGS_CASES_DIR.glob("*.json")) +def _load_dependent_flags_suite() -> tuple[dict[str, Any], list[Path]]: + """Load the dependent-flags suite when it is present in the corpus.""" + case_files = sorted(DEPENDENT_FLAGS_CASES_DIR.glob("*.json")) + if not case_files: + return {}, [] + return json.loads((DEPENDENT_FLAGS_DIR / "ufc-config.json").read_text()), case_files # Load fixture at module level for reuse across tests @@ -76,8 +79,7 @@ def _get_dependent_flags_test_case_files() -> list[Path]: for case in json.loads((FFE_EVALUATION_CASES_DIR / filename).read_text()) if "features" in case["result"] ] -DEPENDENT_FLAGS_UFC_DATA = json.loads((DEPENDENT_FLAGS_DIR / "ufc-config.json").read_text()) -DEPENDENT_FLAGS_TEST_CASE_FILES = _get_dependent_flags_test_case_files() +DEPENDENT_FLAGS_UFC_DATA, DEPENDENT_FLAGS_TEST_CASE_FILES = _load_dependent_flags_suite() DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true",