diff --git a/tests/parametric/test_ffe/dependent-flags/cases/dependent-flags.json b/tests/parametric/test_ffe/dependent-flags/cases/dependent-flags.json deleted file mode 100644 index b91d49d41f6..00000000000 --- a/tests/parametric/test_ffe/dependent-flags/cases/dependent-flags.json +++ /dev/null @@ -1,112 +0,0 @@ -[ - { - "description": "A root flag matches a direct prerequisite by variant key.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-direct-root", - "result": { - "reason": "TARGETING_MATCH", - "value": "direct-match" - }, - "targetingKey": "dependent-direct", - "variationType": "STRING" - }, - { - "description": "ONE_OF does not match when the prerequisite variant key is absent from the allowed list.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-one-of-miss-root", - "result": { - "reason": "STATIC", - "value": "one-of-fallback" - }, - "targetingKey": "dependent-one-of-miss", - "variationType": "STRING" - }, - { - "description": "NOT_ONE_OF does not match when the prerequisite variant key is present in the excluded list.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-not-one-of-hit-root", - "result": { - "reason": "STATIC", - "value": "not-one-of-fallback" - }, - "targetingKey": "dependent-not-one-of-hit", - "variationType": "STRING" - }, - { - "description": "A depth-two tree evaluates its leaf and intermediate prerequisite before matching the root.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-transitive-root", - "result": { - "reason": "TARGETING_MATCH", - "value": "transitive-match" - }, - "targetingKey": "dependent-transitive", - "variationType": "STRING" - }, - { - "description": "A false attribute condition short-circuits before an otherwise fatal missing prerequisite and selects the root fallback.", - "attributes": { - "gate": "closed" - }, - "defaultValue": "caller-default", - "flag": "dependent-short-circuit-root", - "result": { - "reason": "STATIC", - "value": "short-circuit-fallback" - }, - "targetingKey": "dependent-short-circuit", - "variationType": "STRING" - }, - { - "description": "Two membership conditions against the same prerequisite evaluate successfully.", - "attributes": {}, - "defaultValue": "caller-default", - "flag": "dependent-memoized-root", - "result": { - "reason": "TARGETING_MATCH", - "value": "memoized-match" - }, - "targetingKey": "dependent-memoized", - "variationType": "STRING" - }, - { - "description": "A successful prerequisite followed by a missing prerequisite aborts the graph and returns the caller default.", - "attributes": {}, - "defaultValue": "atomic-default", - "flag": "dependent-atomic-error-root", - "result": { - "reason": "ERROR", - "value": "atomic-default" - }, - "targetingKey": "dependent-atomic-error", - "variationType": "STRING" - }, - { - "description": "With maxDependencyDepth 2, a third dependency edge aborts the graph and returns the caller default.", - "attributes": {}, - "defaultValue": "depth-default", - "flag": "dependent-depth-three-root", - "result": { - "reason": "ERROR", - "value": "depth-default" - }, - "targetingKey": "dependent-over-depth", - "variationType": "STRING" - }, - { - "description": "A dependency cycle aborts the graph and returns the caller default.", - "attributes": {}, - "defaultValue": "cycle-default", - "flag": "dependent-cycle-a", - "result": { - "reason": "ERROR", - "value": "cycle-default" - }, - "targetingKey": "dependent-cycle", - "variationType": "STRING" - } -] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index 50d004210d7..ec63232bc34 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -3,7 +3,7 @@ These files are copied from the canonical FFE fixture repository. Canonical source: https://github.com/DataDog/ffe-system-test-data -Source commit: caa6d1ac3e5fbda0ce57ef3ffff3772901f3f466 +Source commit: c8cc8da239341748dc8c30c2145c0f0b5432c958 Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in ffe-system-test-data first, then refresh this snapshot. diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json new file mode 100644 index 00000000000..44e92da23f5 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/cases/dependent-flags.json @@ -0,0 +1,343 @@ +[ + { + "description": "A root flag matches a direct prerequisite by variant key.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-direct-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "direct-match" + }, + "targetingKey": "dependent-direct", + "variationType": "STRING" + }, + { + "description": "ONE_OF does not match when the prerequisite variant key is absent from the allowed list. The prerequisite retains its evaluation event but not its exposure because the containing allocation is rejected.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-one-of-miss-root", + "result": { + "reason": "STATIC", + "value": "one-of-fallback" + }, + "targetingKey": "dependent-one-of-miss", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-one-of-miss-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-one-of-miss-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "NOT_ONE_OF does not match when the prerequisite variant key is present in the excluded list. The prerequisite retains its evaluation event but not its exposure because the containing allocation is rejected.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-not-one-of-hit-root", + "result": { + "reason": "STATIC", + "value": "not-one-of-fallback" + }, + "targetingKey": "dependent-not-one-of-hit", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-not-one-of-hit-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-not-one-of-hit-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "A depth-two tree evaluates its leaf and intermediate prerequisite before matching the root.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-transitive-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "transitive-match" + }, + "targetingKey": "dependent-transitive", + "variationType": "STRING" + }, + { + "description": "A false attribute condition short-circuits before an otherwise fatal missing prerequisite and selects the root fallback.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-short-circuit-root", + "result": { + "reason": "STATIC", + "value": "short-circuit-fallback" + }, + "targetingKey": "dependent-short-circuit", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-short-circuit-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-short-circuit-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "A prerequisite evaluated before a later false condition retains its evaluation event but not its exposure because the containing targeting rule is not selected.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-inapplicable-rule-root", + "result": { + "reason": "STATIC", + "value": "inapplicable-rule-fallback" + }, + "targetingKey": "dependent-inapplicable-rule", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-inapplicable-rule-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-inapplicable-rule-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "When every condition matches, the selected rule retains both the prerequisite and root exposures.", + "attributes": { + "gate": "open" + }, + "defaultValue": "caller-default", + "flag": "dependent-inapplicable-rule-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "inapplicable-rule-match" + }, + "targetingKey": "dependent-applicable-rule", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "flag": "dependent-leaf-on" + }, + { + "allocation": "prerequisite-before-false-condition", + "flag": "dependent-inapplicable-rule-root", + "variant": "matched" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-inapplicable-rule-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "A prerequisite first used by a rejected rule remains eligible for one exposure when a later selected rule reuses it.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-reuse-after-rejection-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "reuse-after-rejection-match" + }, + "targetingKey": "dependent-reuse-after-rejection", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "_count": 1, + "flag": "dependent-leaf-on" + }, + { + "allocation": "selected-reuse", + "flag": "dependent-reuse-after-rejection-root", + "variant": "matched" + } + ], + "evaluationEvents": [ + { + "_count": 1, + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-reuse-after-rejection-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "When a later condition rejects a rule after a transitive dependency succeeds, neither the parent nor leaf exposure remains eligible.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-transitive-inapplicable-rule-root", + "result": { + "reason": "STATIC", + "value": "transitive-inapplicable-fallback" + }, + "targetingKey": "dependent-transitive-inapplicable-rule", + "variationType": "STRING", + "expectations": { + "exposures": [ + { + "allocation": "fallback", + "flag": "dependent-transitive-inapplicable-rule-root", + "variant": "fallback" + } + ], + "evaluationEvents": [ + { + "flag": "dependent-leaf-on", + "errorCode": null + }, + { + "flag": "dependent-transitive-parent", + "errorCode": null + }, + { + "flag": "dependent-transitive-inapplicable-rule-root", + "errorCode": null + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "Two membership conditions against the same prerequisite evaluate successfully.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-memoized-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "memoized-match" + }, + "targetingKey": "dependent-memoized", + "variationType": "STRING" + }, + { + "description": "A reached prerequisite that produces no assignment aborts the graph before a later unconditional root allocation can return or publish an exposure.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-no-assignment-root", + "result": { + "reason": "ERROR", + "errorCode": "GENERAL", + "value": "caller-default" + }, + "targetingKey": "dependent-no-assignment", + "variationType": "STRING", + "expectations": { + "exposures": [], + "evaluationEvents": [ + { + "flag": "dependent-no-assignment-parent", + "errorCode": "GENERAL" + }, + { + "flag": "dependent-no-assignment-root", + "errorCode": "GENERAL" + } + ], + "noUnmatchedEvents": true + } + }, + { + "description": "A successful prerequisite followed by a missing prerequisite aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "atomic-default", + "flag": "dependent-atomic-error-root", + "result": { + "reason": "ERROR", + "value": "atomic-default" + }, + "targetingKey": "dependent-atomic-error", + "variationType": "STRING" + }, + { + "description": "With maxDependencyDepth 2, a third dependency edge aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "depth-default", + "flag": "dependent-depth-three-root", + "result": { + "reason": "ERROR", + "value": "depth-default" + }, + "targetingKey": "dependent-over-depth", + "variationType": "STRING" + }, + { + "description": "A dependency cycle aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "cycle-default", + "flag": "dependent-cycle-a", + "result": { + "reason": "ERROR", + "value": "cycle-default" + }, + "targetingKey": "dependent-cycle", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/dependent-flags/ufc-config.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json similarity index 67% rename from tests/parametric/test_ffe/dependent-flags/ufc-config.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json index d45b41c8c6b..dd15f9346cc 100644 --- a/tests/parametric/test_ffe/dependent-flags/ufc-config.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-suites/dependent-flags/ufc-config.json @@ -427,6 +427,208 @@ } ] }, + "dependent-inapplicable-rule-root": { + "key": "dependent-inapplicable-rule-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "inapplicable-rule-match" + }, + "fallback": { + "key": "fallback", + "value": "inapplicable-rule-fallback" + } + }, + "allocations": [ + { + "key": "prerequisite-before-false-condition", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-reuse-after-rejection-root": { + "key": "dependent-reuse-after-rejection-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "reuse-after-rejection-match" + }, + "fallback": { + "key": "fallback", + "value": "reuse-after-rejection-fallback" + } + }, + "allocations": [ + { + "key": "rejected-first-use", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "selected-reuse", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-transitive-inapplicable-rule-root": { + "key": "dependent-transitive-inapplicable-rule-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "transitive-inapplicable-fallback" + } + }, + "allocations": [ + { + "key": "transitive-prerequisite-before-false-condition", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-transitive-parent" + }, + "operator": "ONE_OF", + "value": [ + "parent-on" + ] + }, + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, "dependent-memoized-root": { "key": "dependent-memoized-root", "enabled": true, @@ -474,6 +676,70 @@ } ] }, + "dependent-no-assignment-parent": { + "key": "dependent-no-assignment-parent", + "enabled": true, + "variationType": "STRING", + "variations": { + "assigned": { + "key": "assigned", + "value": "must-never-be-returned-no-assignment-parent" + } + }, + "allocations": [] + }, + "dependent-no-assignment-root": { + "key": "dependent-no-assignment-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned-no-assignment-match" + }, + "fallback": { + "key": "fallback", + "value": "must-not-be-returned-no-assignment-fallback" + } + }, + "allocations": [ + { + "key": "parent-assigned", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-no-assignment-parent" + }, + "operator": "ONE_OF", + "value": [ + "assigned" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "must-not-be-reached-fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, "dependent-atomic-error-root": { "key": "dependent-atomic-error-root", "enabled": true, diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 48088117087..b35d3280840 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -23,7 +23,7 @@ FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 FFE_SYSTEM_TEST_DATA_DIR = Path(__file__).parent / "ffe-system-test-data" FFE_EVALUATION_CASES_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-cases" -DEPENDENT_FLAGS_DIR = Path(__file__).parent / "dependent-flags" +DEPENDENT_FLAGS_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-suites" / "dependent-flags" DEPENDENT_FLAGS_CASES_DIR = DEPENDENT_FLAGS_DIR / "cases" parametrize = pytest.mark.parametrize @@ -62,9 +62,12 @@ def _filter_test_cases_for_platform(test_cases: list[dict[str, Any]], platform: return applicable_cases -def _get_dependent_flags_test_case_files() -> list[Path]: - """Get the test case files owned by the dependent-flags suite.""" - return sorted(DEPENDENT_FLAGS_CASES_DIR.glob("*.json")) +def _load_dependent_flags_suite() -> tuple[dict[str, Any], list[Path]]: + """Load the dependent-flags suite when it is present in the corpus.""" + case_files = sorted(DEPENDENT_FLAGS_CASES_DIR.glob("*.json")) + if not case_files: + return {}, [] + return json.loads((DEPENDENT_FLAGS_DIR / "ufc-config.json").read_text()), case_files # Load fixture at module level for reuse across tests @@ -76,8 +79,7 @@ def _get_dependent_flags_test_case_files() -> list[Path]: for case in json.loads((FFE_EVALUATION_CASES_DIR / filename).read_text()) if "features" in case["result"] ] -DEPENDENT_FLAGS_UFC_DATA = json.loads((DEPENDENT_FLAGS_DIR / "ufc-config.json").read_text()) -DEPENDENT_FLAGS_TEST_CASE_FILES = _get_dependent_flags_test_case_files() +DEPENDENT_FLAGS_UFC_DATA, DEPENDENT_FLAGS_TEST_CASE_FILES = _load_dependent_flags_suite() DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", @@ -158,6 +160,58 @@ def _ffe_evaluate_with_rc_retry( return result +def _assert_evaluation_cases( + test_case_file: Path, + ufc_data: dict[str, Any], + test_agent: TestAgentAPI, + test_library: APMLibrary, +) -> None: + with test_case_file.open() as fixture_file: + test_cases = json.load(fixture_file) + assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file.name}" + assert test_cases, f"No evaluation cases in {test_case_file.name}" + + platform = context.library.name + test_cases = _filter_test_cases_for_platform(test_cases, platform) + if not test_cases: + pytest.skip(f"No cases in {test_case_file.name} apply to platform {platform}") + + _set_and_wait_ffe_rc(test_agent, ufc_data) + assert test_library.ffe_start(ufc_data), "Failed to start FFE provider" + + for index, test_case in enumerate(test_cases): + expected_result = test_case["result"] + result = _ffe_evaluate_with_rc_retry( + test_library, + flag=test_case["flag"], + variation_type=test_case["variationType"], + default_value=test_case["defaultValue"], + targeting_key=test_case["targetingKey"], + attributes=test_case.get("attributes", {}), + ) + assert not _is_ffe_waiting_for_rc(result), ( + f"Test case {index} in {test_case_file.name} failed: FFE provider did not load RC data after " + f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" + ) + for field in ("value", "reason"): + assert result.get(field) == expected_result[field], ( + f"Test case {index} in {test_case_file.name} failed: " + f"flag='{test_case['flag']}', targetingKey='{test_case['targetingKey']}', field='{field}', " + f"expected={expected_result[field]!r}, actual={result.get(field)!r}" + ) + if "errorCode" in expected_result: + assert result.get("errorCode") == expected_result["errorCode"], ( + f"Test case {index} in {test_case_file.name} failed: " + f"flag='{test_case['flag']}', targetingKey='{test_case['targetingKey']}', field='errorCode', " + f"expected={expected_result['errorCode']!r}, actual={result.get('errorCode')!r}" + ) + elif expected_result["reason"] != "ERROR": + assert result.get("errorCode") in {None, ""}, ( + f"Test case {index} in {test_case_file.name} failed: " + f"flag='{test_case['flag']}' unexpectedly returned errorCode={result.get('errorCode')!r}" + ) + + @scenarios.parametric @features.feature_flags_dynamic_evaluation class Test_Feature_Flag_Dynamic_Evaluation: @@ -188,66 +242,8 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI 4. Handles user targeting, attribute matching, and rollout percentages """ - # Load the test case file test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file - - with test_case_path.open() as f: - test_cases = json.load(f) - assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" - assert test_cases, f"No evaluation cases in {test_case_file}" - - platform = context.library.name - test_cases = _filter_test_cases_for_platform(test_cases, platform) - if not test_cases: - pytest.skip(f"No cases in {test_case_file} apply to platform {platform}") - - # Set up UFC Remote Config and wait for it to be applied - _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) - - # Initialize FFE provider - success = test_library.ffe_start(UFC_FIXTURE_DATA) - assert success, "Failed to start FFE provider" - - # Run each test case - for i, test_case in enumerate(test_cases): - flag = test_case["flag"] - variation_type = test_case["variationType"] - default_value = test_case["defaultValue"] - targeting_key = test_case["targetingKey"] - attributes = test_case.get("attributes", {}) - expected_result = test_case["result"] - - result = _ffe_evaluate_with_rc_retry( - test_library, - flag=flag, - variation_type=variation_type, - default_value=default_value, - targeting_key=targeting_key, - attributes=attributes, - ) - assert not _is_ffe_waiting_for_rc(result), ( - f"Test case {i} in {test_case_file} failed: FFE provider did not load RC data after " - f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" - ) - actual_value = result.get("value") - - # Assert the evaluation result matches expected value - assert actual_value == expected_result["value"], ( - f"Test case {i} in {test_case_file} failed: " - f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={result}" - ) - assert result.get("reason") == expected_result["reason"], ( - f"Test case {i} in {test_case_file}: expected reason={expected_result['reason']}, result={result}" - ) - if "errorCode" in expected_result: - assert result.get("errorCode") == expected_result["errorCode"], ( - f"Test case {i} in {test_case_file}: expected errorCode={expected_result['errorCode']}, result={result}" - ) - elif expected_result["reason"] != "ERROR": - assert result.get("errorCode") in {None, ""}, ( - f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" - ) + _assert_evaluation_cases(test_case_path, UFC_FIXTURE_DATA, test_agent, test_library) @parametrize("library_env", [{**DEFAULT_ENVVARS}]) @parametrize("test_case_file", DEPENDENT_FLAGS_TEST_CASE_FILES) @@ -257,44 +253,8 @@ def test_ffe_dependent_flags( test_agent: TestAgentAPI, test_library: APMLibrary, ) -> None: - """Test dependent flags using only the fixtures in the dependent-flags suite.""" - with test_case_file.open() as f: - test_cases = json.load(f) - - _set_and_wait_ffe_rc(test_agent, DEPENDENT_FLAGS_UFC_DATA) - - success = test_library.ffe_start(DEPENDENT_FLAGS_UFC_DATA) - assert success, "Failed to start FFE provider" - - for i, test_case in enumerate(test_cases): - flag = test_case["flag"] - targeting_key = test_case["targetingKey"] - expected_result = test_case["result"] - result = _ffe_evaluate_with_rc_retry( - test_library, - flag=flag, - variation_type=test_case["variationType"], - default_value=test_case["defaultValue"], - targeting_key=targeting_key, - attributes=test_case.get("attributes", {}), - ) - assert not _is_ffe_waiting_for_rc(result), ( - f"Test case {i} in {test_case_file.name} failed: FFE provider did not load RC data after " - f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" - ) - for field in ("value", "reason", "errorCode"): - if field not in expected_result: - continue - assert result.get(field) == expected_result[field], ( - f"Test case {i} in {test_case_file.name} failed: " - f"flag='{flag}', targetingKey='{targeting_key}', field='{field}', " - f"expected={expected_result[field]!r}, actual={result.get(field)!r}" - ) - if "errorCode" not in expected_result and expected_result.get("reason") != "ERROR": - assert result.get("errorCode") in {None, ""}, ( - f"Test case {i} in {test_case_file.name} failed: " - f"flag='{flag}' unexpectedly returned errorCode={result.get('errorCode')!r}" - ) + """Run the imported dependent-flags feature suite through the shared evaluator harness.""" + _assert_evaluation_cases(test_case_file, DEPENDENT_FLAGS_UFC_DATA, test_agent, test_library) @scenarios.parametric diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py index e7f3b95f484..9b9fbf2289e 100644 --- a/utils/scripts/update_ffe_fixtures.py +++ b/utils/scripts/update_ffe_fixtures.py @@ -64,15 +64,17 @@ def _copy_entry(source: Path, destination: Path) -> None: def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: - # Import only the server evaluator corpus that this suite consumes. Other - # upstream suites (for example precomputed assignments) have different formats. + # Import only rules-based evaluator corpora. Precomputed assignments use a + # different format and are consumed by a different test harness. for name in ("ufc-config.json", "evaluation-cases"): _copy_entry(source / name, snapshot / name) + feature_suites = source / "evaluation-suites" + if feature_suites.exists() or feature_suites.is_symlink(): + _copy_entry(feature_suites, snapshot / feature_suites.name) -def _validate_fixture_snapshot(snapshot: Path) -> int: - config_path = snapshot / "ufc-config.json" - cases_directory = snapshot / "evaluation-cases" + +def _validate_evaluation_suite(config_path: Path, cases_directory: Path) -> int: if not config_path.is_file() or not cases_directory.is_dir(): msg = "FFE fixture repository does not contain the expected fixture layout" raise ValueError(msg) @@ -91,7 +93,7 @@ def _validate_fixture_snapshot(snapshot: Path) -> int: msg = "No FFE JSON fixture files found" raise ValueError(msg) - fixture_count = 0 + case_count = 0 for case_file in case_files: with case_file.open(encoding="utf-8") as fixture_file: test_cases = json.load(fixture_file) @@ -101,11 +103,35 @@ def _validate_fixture_snapshot(snapshot: Path) -> int: if not test_cases: msg = f"{case_file} must contain at least one evaluation case" raise ValueError(msg) - fixture_count += len(test_cases) + case_count += len(test_cases) + + return case_count - if fixture_count == 0: - msg = "No FFE fixture test cases found" + +def _validate_fixture_snapshot(snapshot: Path) -> int: + fixture_count = _validate_evaluation_suite(snapshot / "ufc-config.json", snapshot / "evaluation-cases") + + suites_directory = snapshot / "evaluation-suites" + if not suites_directory.exists(): + return fixture_count + if not suites_directory.is_dir() or suites_directory.is_symlink(): + msg = "FFE evaluation-suites must be a directory" + raise ValueError(msg) + suite_directories = sorted(suites_directory.iterdir()) + if not suite_directories: + msg = "FFE evaluation-suites must contain at least one feature suite" raise ValueError(msg) + for suite_directory in suite_directories: + if not suite_directory.is_dir() or suite_directory.is_symlink(): + msg = "FFE evaluation-suites must contain only suite directories" + raise ValueError(msg) + if {entry.name for entry in suite_directory.iterdir()} != {"ufc-config.json", "cases"}: + msg = f"FFE feature suite {suite_directory.name} must contain only ufc-config.json and cases" + raise ValueError(msg) + fixture_count += _validate_evaluation_suite( + suite_directory / "ufc-config.json", + suite_directory / "cases", + ) return fixture_count