From 7691de4829951a3b2814099e17bf0f493c3d4556 Mon Sep 17 00:00:00 2001 From: Helen Han Date: Tue, 3 Nov 2026 13:35:19 -0500 Subject: [PATCH 1/5] test(ffe): enforce dependent flag RC capability contract Check bit 50 on every FFE RC poll before delivery and after updates, exercise variant-key prerequisites through Agent RC, and cover a known legacy SDK without support. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- docs/understand/scenarios/parametric.md | 34 +++++ manifests/cpp.yml | 2 + manifests/dotnet.yml | 2 + manifests/golang.yml | 2 + manifests/java.yml | 2 + manifests/nodejs.yml | 2 + manifests/php.yml | 2 + manifests/python.yml | 4 + manifests/ruby.yml | 2 + manifests/rust.yml | 2 + .../test_ffe/test_dependent_flags_rc.py | 141 ++++++++++++++++++ tests/parametric/test_ffe/utils.py | 12 ++ .../test_the_test/test_ffe_rc_capabilities.py | 39 +++++ 13 files changed, 246 insertions(+) create mode 100644 tests/parametric/test_ffe/test_dependent_flags_rc.py create mode 100644 tests/test_the_test/test_ffe_rc_capabilities.py diff --git a/docs/understand/scenarios/parametric.md b/docs/understand/scenarios/parametric.md index 4a901b56c0b..4bc87807fcd 100644 --- a/docs/understand/scenarios/parametric.md +++ b/docs/understand/scenarios/parametric.md @@ -308,3 +308,37 @@ def test_stuff(test_library): [2]: https://docs.pytest.org/en/6.2.x/usage.html#specifying-tests-selecting-tests [3]: https://github.com/roydahan/xunit-viewer [4]: ../../utils/build/docker/cpp/parametric/CMakeLists.txt + +### Dependent flags over Remote Configuration + +`tests/parametric/test_ffe/test_dependent_flags_rc.py` checks the wire contract for +server SDKs using Agent RC. The supported-build test requires bit 50 on every +observed request subscribing to `FFE_FLAGS`, before any dependent configuration +is delivered and after configuration updates. It verifies prerequisite variant +keys rather than flag values, using configuration delivered exclusively by RC. +The test agent models capability-gated delivery; this does not validate production +backend targeting. + +Use the [custom tracer build instructions](../../execute/binaries.md) to select +the SDK and native library under test. With an unreleased supported build, force +the positive class so a missing-feature manifest cannot turn a regression into +an expected failure: + +```sh +TEST_LIBRARY=python ./run.sh PARAMETRIC \ + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC \ + -F tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC +``` + +For a build known to lack dependent-flag support, select and force +`Test_FFE_Without_Dependent_Flags_RC` instead. It must receive and evaluate an +ordinary RC flag while advertising no bit 50. No RC traffic is a failure, not +proof that an unsupported build is safe. Python 4.16.0 is the explicit legacy +baseline in the manifest; other legacy/native combinations can use `-F`. + +Enable the positive class in each SDK's manifest when its dependent-flag support +ships, alongside the evaluation suite. Python's test targets the tested 4.17 +development line and must land with its SDK implementation. Other SDKs retain +missing-feature declarations until their supporting native packages ship. For +Ruby, verify the loaded native package as well as the SDK version. Do not infer +expected support from whether the SDK happens to advertise bit 50. diff --git a/manifests/cpp.yml b/manifests/cpp.yml index bc2ba4b839d..e6dec7de8c9 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -187,6 +187,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_exposure_hooks.py: incomplete_test_app (Exposure hook observations are not implemented) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index dbbad51aee5..bd482146553 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1218,6 +1218,8 @@ manifest: : # TODO: a lower version might be supported - declaration: bug (FFL-3325) component_version: <3.55.0 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script diff --git a/manifests/golang.yml b/manifests/golang.yml index 1bed7c35590..7811357e6d6 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2185,6 +2185,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/java.yml b/manifests/java.yml index 08b1f162174..c2dcb1dfb7e 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4266,6 +4266,8 @@ manifest: ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_stable_true_preserves_default_agentless : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: v1.69.0-SNAPSHOT diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index a22f77adf91..ecaff15ff09 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2700,6 +2700,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: *ref_5_47_0 tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: *ref_5_56_0 tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/php.yml b/manifests/php.yml index a0b173d64df..0890e93235c 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1649,6 +1649,8 @@ manifest: ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_stable_true_preserves_default_agentless : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/python.yml b/manifests/python.yml index 7f14c4e2e6e..025c148b3be 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -3153,6 +3153,10 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: v2.20.0 tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: v4.17.0-dev + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: + - declaration: irrelevant (Legacy coverage is pinned to Python 4.16.0) + component_version: '<4.16.0 || >4.16.0' tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index ce005e438f3..cd6c9d5af88 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2711,6 +2711,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/rust.yml b/manifests/rust.yml index 8de4581130c..03926dc6167 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -299,6 +299,8 @@ manifest: tests/parametric/test_dynamic_configuration.py::Test_DynamicConfigSdkConfiguration::test_sdk_config_profiling_enabled: missing_feature (SDK_CONFIGURATION capability not yet implemented) tests/parametric/test_extract_behavior.py: '>=0.4.1-dev' tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature diff --git a/tests/parametric/test_ffe/test_dependent_flags_rc.py b/tests/parametric/test_ffe/test_dependent_flags_rc.py new file mode 100644 index 00000000000..d436741a68f --- /dev/null +++ b/tests/parametric/test_ffe/test_dependent_flags_rc.py @@ -0,0 +1,141 @@ +"""Dependent-flag capability and evaluation contract on the Agent RC transport. + +Supported and legacy builds have separate manifest-controlled expectations. Never +infer support from the advertised bit: doing so would hide a missing-bit regression. +""" + +import time +from typing import Any + +from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import assert_dependent_flag_capability +from utils import features, pytest, scenarios +from utils.dd_constants import RemoteConfigApplyState +from utils.docker_fixtures import TestAgentAPI + +RC_PRODUCT = "FFE_FLAGS" +ENV = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", +} + + +def _ffe_requests(test_agent: TestAgentAPI) -> list[dict[str, Any]]: + return [ + request + for request in test_agent.rc_requests(post_only=True) + if RC_PRODUCT in request["body"]["client"].get("products", []) + ] + + +def _observe_polls(test_agent: TestAgentAPI, *, supported: bool, minimum: int = 3) -> None: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + requests = _ffe_requests(test_agent) + if requests: + assert_dependent_flag_capability(requests, supported=supported) + if len(requests) >= minimum: + return + time.sleep(0.1) + raise AssertionError(f"Expected at least {minimum} FFE_FLAGS RC polls, got {len(requests)}") + + +def _config(*, parent_variant: str, dependent: bool) -> dict[str, Any]: + parent = { + "key": "parent", + "enabled": True, + "variationType": "STRING", + # The selected variant key deliberately differs from its value. + "variations": { + "approved": {"key": "approved", "value": "not-the-variant-key"}, + "rejected": {"key": "rejected", "value": "approved"}, + }, + "allocations": [ + {"key": "parent-default", "splits": [{"variationKey": parent_variant, "shards": []}], "doLog": False} + ], + } + flags = {"parent": parent} + if dependent: + flags["child"] = { + "key": "child", + "enabled": True, + "variationType": "STRING", + "variations": {key: {"key": key, "value": key} for key in ("matched", "unmatched")}, + "allocations": [ + { + "key": "prerequisite", + "rules": [ + { + "conditions": [ + {"flagEvaluation": {"key": "parent"}, "operator": "ONE_OF", "value": ["approved"]} + ] + } + ], + "splits": [{"variationKey": "matched", "shards": []}], + "doLog": False, + }, + {"key": "fallback", "splits": [{"variationKey": "unmatched", "shards": []}], "doLog": False}, + ], + } + return {"createdAt": "2026-11-03T00:00:00Z", "format": "SERVER", "environment": {"name": "test"}, "flags": flags} + + +def _deliver(test_agent: TestAgentAPI, *, config_id: str, parent_variant: str, dependent: bool) -> None: + test_agent.set_remote_config( + path=f"datadog/2/{RC_PRODUCT}/{config_id}/config", + payload=_config(parent_variant=parent_variant, dependent=dependent), + ) + test_agent.wait_for_rc_apply_state( + RC_PRODUCT, + state=RemoteConfigApplyState.ACKNOWLEDGED, + config_id=config_id, + wait_loops=1000, + ) + + +def _assert_value(test_library: APMLibrary, flag: str, value: str) -> None: + deadline = time.monotonic() + 5 + while True: + result = test_library.ffe_evaluate( + flag=flag, + variation_type="STRING", + default_value="caller-default", + targeting_key="subject", + ) + if result.get("value") == value and result.get("errorCode") in (None, ""): + return + assert time.monotonic() < deadline, result + time.sleep(0.1) + + +@scenarios.parametric +@features.feature_flags_dynamic_evaluation +class Test_FFE_Dependent_Flags_RC: + @pytest.mark.parametrize("library_env", [ENV]) + def test_capability_and_gated_delivery(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """Advertise before delivery, evaluate variant keys, and retain the bit after updates.""" + assert test_library.ffe_start(), "Failed to initialize the provider" + # No inline configuration: only RC can deliver these flag definitions. + _observe_polls(test_agent, supported=True) + for variant, expected in (("approved", "matched"), ("rejected", "unmatched")): + assert_dependent_flag_capability(_ffe_requests(test_agent), supported=True) + _deliver(test_agent, config_id="dependent-flags", parent_variant=variant, dependent=True) + _assert_value(test_library, "child", expected) + previous_count = len(_ffe_requests(test_agent)) + _observe_polls(test_agent, supported=True, minimum=previous_count + 2) + + +@scenarios.parametric +@features.feature_flags_dynamic_evaluation +class Test_FFE_Without_Dependent_Flags_RC: + @pytest.mark.parametrize("library_env", [ENV]) + def test_legacy_build_does_not_advertise(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """A known legacy build receives ordinary flags and never advertises bit 50.""" + assert test_library.ffe_start(), "Failed to initialize the provider" + _observe_polls(test_agent, supported=False) + _deliver(test_agent, config_id="ordinary-flags", parent_variant="approved", dependent=False) + _assert_value(test_library, "parent", "not-the-variant-key") + previous_count = len(_ffe_requests(test_agent)) + _observe_polls(test_agent, supported=False, minimum=previous_count + 2) diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index b82b1c64c0a..335a8586fad 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -175,3 +175,15 @@ def ffe_events_from_requests(requests: Sequence[Mapping[str, Any]], flag: str) - payload = json.loads(body) events[destination].extend(event for event in payload[field] if event.get("flag", {}).get("key") == flag) return events + + +def assert_dependent_flag_capability(requests: list[dict[str, Any]], *, supported: bool) -> None: + assert requests, "No FFE_FLAGS RC requests observed; absence of traffic is not evidence of compatibility" + for request in requests: + client = request["body"]["client"] + raw = client.get("capabilities", "") + decoded = bytes(raw) if isinstance(raw, list) else base64.b64decode(raw) + mask = int.from_bytes(decoded, byteorder="big") + assert bool(mask & (1 << 50)) is supported, ( + f"FFE RC client {client.get('id')} must advertise bit 50={supported}; got {mask:#x}" + ) diff --git a/tests/test_the_test/test_ffe_rc_capabilities.py b/tests/test_the_test/test_ffe_rc_capabilities.py new file mode 100644 index 00000000000..3900bd0c039 --- /dev/null +++ b/tests/test_the_test/test_ffe_rc_capabilities.py @@ -0,0 +1,39 @@ +"""Ensure the RC assertion rejects missing, intermittent, and false advertisements.""" + +import base64 +from typing import Any + +from tests.parametric.test_ffe.utils import assert_dependent_flag_capability +from utils import features, pytest, scenarios + + +def _request(mask: int, *, byte_array: bool) -> dict[str, Any]: + raw = mask.to_bytes(7, byteorder="big") + return { + "body": { + "client": {"id": "ffe-client", "capabilities": list(raw) if byte_array else base64.b64encode(raw).decode()} + } + } + + +@scenarios.test_the_test +@features.not_reported +class Test_FFE_RC_Capability_Assertion: + @pytest.mark.parametrize("byte_array", [False, True]) + def test_supported_and_legacy(self, *, byte_array: bool) -> None: + assert_dependent_flag_capability([_request((1 << 46) | (1 << 50), byte_array=byte_array)], supported=True) + assert_dependent_flag_capability([_request(1 << 46, byte_array=byte_array)], supported=False) + + @pytest.mark.parametrize("byte_array", [False, True]) + def test_missing_bit_fails_even_if_later_poll_recovers(self, *, byte_array: bool) -> None: + requests = [_request(1 << 46, byte_array=byte_array), _request((1 << 46) | (1 << 50), byte_array=byte_array)] + with pytest.raises(AssertionError, match="bit 50=True"): + assert_dependent_flag_capability(requests, supported=True) + + def test_false_advertisement_fails(self) -> None: + with pytest.raises(AssertionError, match="bit 50=False"): + assert_dependent_flag_capability([_request(1 << 50, byte_array=False)], supported=False) + + def test_no_traffic_is_not_a_pass(self) -> None: + with pytest.raises(AssertionError, match="No FFE_FLAGS RC requests"): + assert_dependent_flag_capability([], supported=False) From 43090f4e0dc34aef506c5a344837f87143a11ead Mon Sep 17 00:00:00 2001 From: "helen.han" Date: Wed, 4 Nov 2026 18:25:32 +0000 Subject: [PATCH 2/5] test(ffe): defer Python dependent flag capability activation Keep the positive RC capability test missing_feature until a Python release containing dd-trace-py #21346 is confirmed, and align the activation docs. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- docs/understand/scenarios/parametric.md | 9 +++++---- manifests/python.yml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/understand/scenarios/parametric.md b/docs/understand/scenarios/parametric.md index 4bc87807fcd..7ef30ebf0e2 100644 --- a/docs/understand/scenarios/parametric.md +++ b/docs/understand/scenarios/parametric.md @@ -337,8 +337,9 @@ proof that an unsupported build is safe. Python 4.16.0 is the explicit legacy baseline in the manifest; other legacy/native combinations can use `-F`. Enable the positive class in each SDK's manifest when its dependent-flag support -ships, alongside the evaluation suite. Python's test targets the tested 4.17 -development line and must land with its SDK implementation. Other SDKs retain -missing-feature declarations until their supporting native packages ship. For -Ruby, verify the loaded native package as well as the SDK version. Do not infer +ships, alongside the evaluation suite. All SDKs retain missing-feature declarations +until a release containing dependent-flag support is confirmed. Python's +implementation is tracked in [dd-trace-py #21346](https://github.com/DataDog/dd-trace-py/pull/21346); +use the forced positive class to test that build before enabling a release threshold. +For Ruby, verify the loaded native package as well as the SDK version. Do not infer expected support from whether the SDK happens to advertise bit 50. diff --git a/manifests/python.yml b/manifests/python.yml index 025c148b3be..74c6ead2037 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -3153,7 +3153,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: v2.20.0 tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev - tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: v4.17.0-dev + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release; dd-trace-py PR 21346) tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: - declaration: irrelevant (Legacy coverage is pinned to Python 4.16.0) component_version: '<4.16.0 || >4.16.0' From d49323cb1b9177a14919b0a29d324b3f21ac0a73 Mon Sep 17 00:00:00 2001 From: "helen.han" Date: Wed, 4 Nov 2026 18:40:08 +0000 Subject: [PATCH 3/5] test(ffe): use registered RC capability constants Use the shared dependent-flag capability in assertions and capability constants for regression fixtures instead of hardcoded bit positions. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- tests/parametric/test_ffe/utils.py | 7 +++++-- .../test_the_test/test_ffe_rc_capabilities.py | 21 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index 335a8586fad..dcd52de58ac 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -17,6 +17,8 @@ from pathlib import Path from typing import Any +from utils.dd_constants import Capabilities + # Varint encoding constants VARINT_DATA_BITS = 0x7F # Lower 7 bits contain data VARINT_CONTINUATION_BIT = 0x80 # MSB indicates more bytes follow @@ -179,11 +181,12 @@ def ffe_events_from_requests(requests: Sequence[Mapping[str, Any]], flag: str) - def assert_dependent_flag_capability(requests: list[dict[str, Any]], *, supported: bool) -> None: assert requests, "No FFE_FLAGS RC requests observed; absence of traffic is not evidence of compatibility" + capability = Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1 for request in requests: client = request["body"]["client"] raw = client.get("capabilities", "") decoded = bytes(raw) if isinstance(raw, list) else base64.b64decode(raw) mask = int.from_bytes(decoded, byteorder="big") - assert bool(mask & (1 << 50)) is supported, ( - f"FFE RC client {client.get('id')} must advertise bit 50={supported}; got {mask:#x}" + assert bool(mask & (1 << capability)) is supported, ( + f"FFE RC client {client.get('id')} must advertise bit {capability}={supported}; got {mask:#x}" ) diff --git a/tests/test_the_test/test_ffe_rc_capabilities.py b/tests/test_the_test/test_ffe_rc_capabilities.py index 3900bd0c039..1c6b77fa614 100644 --- a/tests/test_the_test/test_ffe_rc_capabilities.py +++ b/tests/test_the_test/test_ffe_rc_capabilities.py @@ -5,6 +5,10 @@ from tests.parametric.test_ffe.utils import assert_dependent_flag_capability from utils import features, pytest, scenarios +from utils.dd_constants import Capabilities + +DEPENDENT_FLAG_MASK = 1 << Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1 +FLAG_CONFIGURATION_MASK = 1 << Capabilities.FFE_FLAG_CONFIGURATION_RULES def _request(mask: int, *, byte_array: bool) -> dict[str, Any]: @@ -21,18 +25,23 @@ def _request(mask: int, *, byte_array: bool) -> dict[str, Any]: class Test_FFE_RC_Capability_Assertion: @pytest.mark.parametrize("byte_array", [False, True]) def test_supported_and_legacy(self, *, byte_array: bool) -> None: - assert_dependent_flag_capability([_request((1 << 46) | (1 << 50), byte_array=byte_array)], supported=True) - assert_dependent_flag_capability([_request(1 << 46, byte_array=byte_array)], supported=False) + assert_dependent_flag_capability( + [_request(FLAG_CONFIGURATION_MASK | DEPENDENT_FLAG_MASK, byte_array=byte_array)], supported=True + ) + assert_dependent_flag_capability([_request(FLAG_CONFIGURATION_MASK, byte_array=byte_array)], supported=False) @pytest.mark.parametrize("byte_array", [False, True]) def test_missing_bit_fails_even_if_later_poll_recovers(self, *, byte_array: bool) -> None: - requests = [_request(1 << 46, byte_array=byte_array), _request((1 << 46) | (1 << 50), byte_array=byte_array)] - with pytest.raises(AssertionError, match="bit 50=True"): + requests = [ + _request(FLAG_CONFIGURATION_MASK, byte_array=byte_array), + _request(FLAG_CONFIGURATION_MASK | DEPENDENT_FLAG_MASK, byte_array=byte_array), + ] + with pytest.raises(AssertionError, match=f"bit {Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1}=True"): assert_dependent_flag_capability(requests, supported=True) def test_false_advertisement_fails(self) -> None: - with pytest.raises(AssertionError, match="bit 50=False"): - assert_dependent_flag_capability([_request(1 << 50, byte_array=False)], supported=False) + with pytest.raises(AssertionError, match=f"bit {Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1}=False"): + assert_dependent_flag_capability([_request(DEPENDENT_FLAG_MASK, byte_array=False)], supported=False) def test_no_traffic_is_not_a_pass(self) -> None: with pytest.raises(AssertionError, match="No FFE_FLAGS RC requests"): From d863e930a1bea126acfc467b75a242541a32427f Mon Sep 17 00:00:00 2001 From: "helen.han" Date: Thu, 5 Nov 2026 22:04:44 +0000 Subject: [PATCH 4/5] test(ffe): fix RC polling, update ACKs, and cold startup Include GET polls, require changed cached content with its ACK, and coordinate provider initialization with capability-gated RC delivery. Add harness regressions and bound the startup HTTP request. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- .../parametric/test_ffe/dependent_flags_rc.py | 87 +++++++++++++ .../test_ffe/test_dependent_flags_rc.py | 80 +++++------- .../test_the_test/test_ffe_rc_capabilities.py | 117 +++++++++++++++++- .../_test_clients/_test_client_parametric.py | 4 +- 4 files changed, 236 insertions(+), 52 deletions(-) create mode 100644 tests/parametric/test_ffe/dependent_flags_rc.py diff --git a/tests/parametric/test_ffe/dependent_flags_rc.py b/tests/parametric/test_ffe/dependent_flags_rc.py new file mode 100644 index 00000000000..dda4db4c5a1 --- /dev/null +++ b/tests/parametric/test_ffe/dependent_flags_rc.py @@ -0,0 +1,87 @@ +"""Capability-gated RC delivery, including cold provider startup and content updates.""" + +from concurrent.futures import ThreadPoolExecutor +import time +from typing import Any + +from tests.parametric.test_ffe.utils import assert_dependent_flag_capability +from utils.dd_constants import RemoteConfigApplyState +from utils.docker_fixtures import ParametricTestClientApi, TestAgentAPI + +RC_PRODUCT = "FFE_FLAGS" + + +def ffe_requests(test_agent: TestAgentAPI) -> list[dict[str, Any]]: + return [ + request for request in test_agent.rc_requests() if RC_PRODUCT in request["body"]["client"].get("products", []) + ] + + +def observe_polls(test_agent: TestAgentAPI, *, supported: bool, minimum: int = 3) -> None: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + requests = ffe_requests(test_agent) + if requests: + assert_dependent_flag_capability(requests, supported=supported) + if len(requests) >= minimum: + return + time.sleep(0.1) + raise AssertionError(f"Expected at least {minimum} FFE_FLAGS RC polls, got {len(requests)}") + + +def deliver(test_agent: TestAgentAPI, *, config_id: str, payload: dict[str, Any]) -> None: + path = f"datadog/2/{RC_PRODUCT}/{config_id}/config" + previous_requests = test_agent.rc_requests() + previous_target = next( + ( + target + for request in reversed(previous_requests) + for target in request["body"].get("cached_target_files", []) + if target["path"] == path + ), + None, + ) + test_agent.set_remote_config(path=path, payload=payload) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + # A stale ACK for the same ID must not acknowledge new content. Keep all + # requests for capability checks, but require a new target and ACK together. + for request in test_agent.rc_requests()[len(previous_requests) :]: + body = request["body"] + changed = any( + target["path"] == path and target != previous_target for target in body.get("cached_target_files", []) + ) + acknowledged = any( + state["product"] == RC_PRODUCT + and state["id"] == config_id + and state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value + for state in body["client"]["state"].get("config_states", []) + ) + if changed and acknowledged: + return + time.sleep(0.05) + raise AssertionError(f"The SDK did not acknowledge new configuration content for {config_id}") + + +def start_and_deliver( + test_agent: TestAgentAPI, + test_library: ParametricTestClientApi, + *, + config_id: str, + payload: dict[str, Any], + supported: bool, +) -> None: + # .NET waits for configuration inside /ffe/start; Java can return not-ready + # before delivery. Trigger startup without requiring readiness until after RC. + with ThreadPoolExecutor(max_workers=1) as executor: + started = executor.submit(test_library.ffe_start) + observe_polls(test_agent, supported=supported) + deliver(test_agent, config_id=config_id, payload=payload) + if started.result(timeout=10): + return + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if test_library.ffe_start(): + return + time.sleep(0.1) + raise AssertionError("Failed to initialize the provider after RC delivery") diff --git a/tests/parametric/test_ffe/test_dependent_flags_rc.py b/tests/parametric/test_ffe/test_dependent_flags_rc.py index d436741a68f..46d9c508786 100644 --- a/tests/parametric/test_ffe/test_dependent_flags_rc.py +++ b/tests/parametric/test_ffe/test_dependent_flags_rc.py @@ -8,12 +8,16 @@ from typing import Any from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.dependent_flags_rc import ( + deliver, + ffe_requests, + observe_polls, + start_and_deliver, +) from tests.parametric.test_ffe.utils import assert_dependent_flag_capability from utils import features, pytest, scenarios -from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI -RC_PRODUCT = "FFE_FLAGS" ENV = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", "DD_FEATURE_FLAGS_ENABLED": "true", @@ -22,26 +26,6 @@ } -def _ffe_requests(test_agent: TestAgentAPI) -> list[dict[str, Any]]: - return [ - request - for request in test_agent.rc_requests(post_only=True) - if RC_PRODUCT in request["body"]["client"].get("products", []) - ] - - -def _observe_polls(test_agent: TestAgentAPI, *, supported: bool, minimum: int = 3) -> None: - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - requests = _ffe_requests(test_agent) - if requests: - assert_dependent_flag_capability(requests, supported=supported) - if len(requests) >= minimum: - return - time.sleep(0.1) - raise AssertionError(f"Expected at least {minimum} FFE_FLAGS RC polls, got {len(requests)}") - - def _config(*, parent_variant: str, dependent: bool) -> dict[str, Any]: parent = { "key": "parent", @@ -82,19 +66,6 @@ def _config(*, parent_variant: str, dependent: bool) -> dict[str, Any]: return {"createdAt": "2026-11-03T00:00:00Z", "format": "SERVER", "environment": {"name": "test"}, "flags": flags} -def _deliver(test_agent: TestAgentAPI, *, config_id: str, parent_variant: str, dependent: bool) -> None: - test_agent.set_remote_config( - path=f"datadog/2/{RC_PRODUCT}/{config_id}/config", - payload=_config(parent_variant=parent_variant, dependent=dependent), - ) - test_agent.wait_for_rc_apply_state( - RC_PRODUCT, - state=RemoteConfigApplyState.ACKNOWLEDGED, - config_id=config_id, - wait_loops=1000, - ) - - def _assert_value(test_library: APMLibrary, flag: str, value: str) -> None: deadline = time.monotonic() + 5 while True: @@ -116,15 +87,22 @@ class Test_FFE_Dependent_Flags_RC: @pytest.mark.parametrize("library_env", [ENV]) def test_capability_and_gated_delivery(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """Advertise before delivery, evaluate variant keys, and retain the bit after updates.""" - assert test_library.ffe_start(), "Failed to initialize the provider" - # No inline configuration: only RC can deliver these flag definitions. - _observe_polls(test_agent, supported=True) - for variant, expected in (("approved", "matched"), ("rejected", "unmatched")): - assert_dependent_flag_capability(_ffe_requests(test_agent), supported=True) - _deliver(test_agent, config_id="dependent-flags", parent_variant=variant, dependent=True) - _assert_value(test_library, "child", expected) - previous_count = len(_ffe_requests(test_agent)) - _observe_polls(test_agent, supported=True, minimum=previous_count + 2) + start_and_deliver( + test_agent, + test_library, + config_id="dependent-flags", + payload=_config(parent_variant="approved", dependent=True), + supported=True, + ) + _assert_value(test_library, "child", "matched") + previous_count = len(ffe_requests(test_agent)) + observe_polls(test_agent, supported=True, minimum=previous_count + 2) + + assert_dependent_flag_capability(ffe_requests(test_agent), supported=True) + deliver(test_agent, config_id="dependent-flags", payload=_config(parent_variant="rejected", dependent=True)) + _assert_value(test_library, "child", "unmatched") + previous_count = len(ffe_requests(test_agent)) + observe_polls(test_agent, supported=True, minimum=previous_count + 2) @scenarios.parametric @@ -133,9 +111,13 @@ class Test_FFE_Without_Dependent_Flags_RC: @pytest.mark.parametrize("library_env", [ENV]) def test_legacy_build_does_not_advertise(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """A known legacy build receives ordinary flags and never advertises bit 50.""" - assert test_library.ffe_start(), "Failed to initialize the provider" - _observe_polls(test_agent, supported=False) - _deliver(test_agent, config_id="ordinary-flags", parent_variant="approved", dependent=False) + start_and_deliver( + test_agent, + test_library, + config_id="ordinary-flags", + payload=_config(parent_variant="approved", dependent=False), + supported=False, + ) _assert_value(test_library, "parent", "not-the-variant-key") - previous_count = len(_ffe_requests(test_agent)) - _observe_polls(test_agent, supported=False, minimum=previous_count + 2) + previous_count = len(ffe_requests(test_agent)) + observe_polls(test_agent, supported=False, minimum=previous_count + 2) diff --git a/tests/test_the_test/test_ffe_rc_capabilities.py b/tests/test_the_test/test_ffe_rc_capabilities.py index 1c6b77fa614..b99bc12c5b9 100644 --- a/tests/test_the_test/test_ffe_rc_capabilities.py +++ b/tests/test_the_test/test_ffe_rc_capabilities.py @@ -1,11 +1,15 @@ -"""Ensure the RC assertion rejects missing, intermittent, and false advertisements.""" +"""Regressions for RC capability assertions, gated startup, and update acknowledgement.""" import base64 +from threading import Event from typing import Any +from unittest.mock import MagicMock, patch +from tests.parametric.test_ffe.dependent_flags_rc import deliver, ffe_requests, start_and_deliver from tests.parametric.test_ffe.utils import assert_dependent_flag_capability from utils import features, pytest, scenarios -from utils.dd_constants import Capabilities +from utils.dd_constants import Capabilities, RemoteConfigApplyState +from utils.docker_fixtures import TestAgentAPI, ParametricTestClientApi DEPENDENT_FLAG_MASK = 1 << Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1 FLAG_CONFIGURATION_MASK = 1 << Capabilities.FFE_FLAG_CONFIGURATION_RULES @@ -46,3 +50,112 @@ def test_false_advertisement_fails(self) -> None: def test_no_traffic_is_not_a_pass(self) -> None: with pytest.raises(AssertionError, match="No FFE_FLAGS RC requests"): assert_dependent_flag_capability([], supported=False) + + +RC_PATH = "datadog/2/FFE_FLAGS/flags/config" + + +def _poll(*, content: str = "", acknowledged: bool = False, product: str = "FFE_FLAGS") -> dict[str, Any]: + request = _request(FLAG_CONFIGURATION_MASK | DEPENDENT_FLAG_MASK, byte_array=False) + request["method"] = "GET" + request["body"]["client"].update( + products=["FFE_FLAGS"], + state={ + "config_states": [ + { + "id": "flags", + "product": product, + "apply_state": ( + RemoteConfigApplyState.ACKNOWLEDGED.value + if acknowledged + else RemoteConfigApplyState.UNACKNOWLEDGED.value + ), + } + ] + }, + ) + request["body"]["cached_target_files"] = ( + [{"path": RC_PATH, "hashes": [{"algorithm": "sha256", "hash": content}]}] if content else [] + ) + return request + + +@scenarios.test_the_test +@features.not_reported +class Test_FFE_RC_Delivery: + def test_get_and_post_polls_keep_product_filter(self) -> None: + agent = MagicMock(spec=TestAgentAPI) + get = _poll() + post = _poll() + post["method"] = "POST" + unrelated = _poll() + unrelated["body"]["client"]["products"] = ["APM_TRACING"] + agent.rc_requests.side_effect = lambda *, post_only=False: ([post] if post_only else [get, unrelated, post]) + assert ffe_requests(agent) == [get, post] + + def test_update_requires_new_content_and_ack_in_same_poll(self) -> None: + agent = MagicMock(spec=TestAgentAPI) + old = _poll(content="old", acknowledged=True) + new_without_ack = _poll(content="new") + wrong_product = _poll(content="new", acknowledged=True, product="APM_TRACING") + new_ack = _poll(content="new", acknowledged=True) + agent.rc_requests.side_effect = [ + [old], + [old, old], + [old, old, new_without_ack], + [old, old, new_without_ack, wrong_product], + [old, old, new_without_ack, wrong_product, new_ack], + ] + with patch("tests.parametric.test_ffe.dependent_flags_rc.time.sleep"): + deliver(agent, config_id="flags", payload={"flags": {}}) + assert agent.rc_requests.call_count == 5 + agent.clear.assert_not_called() + + def test_stale_ack_times_out(self) -> None: + agent = MagicMock(spec=TestAgentAPI) + old = _poll(content="old", acknowledged=True) + agent.rc_requests.side_effect = [[old], [old, old]] + with ( + patch("tests.parametric.test_ffe.dependent_flags_rc.time.monotonic", side_effect=[0, 0, 11]), + patch("tests.parametric.test_ffe.dependent_flags_rc.time.sleep"), + pytest.raises(AssertionError, match="did not acknowledge new configuration content"), + ): + deliver(agent, config_id="flags", payload={"flags": {}}) + + @pytest.mark.parametrize("blocking", [True, False]) + @pytest.mark.parametrize("supported", [True, False]) + def test_cold_start_delivers_before_requiring_readiness(self, *, blocking: bool, supported: bool) -> None: + agent = MagicMock(spec=TestAgentAPI) + library = MagicMock(spec=ParametricTestClientApi) + starting = Event() + delivered = Event() + observed: list[dict[str, Any]] = [] + + def start() -> bool: + starting.set() + if blocking: + assert delivered.wait(timeout=5), "Startup blocked waiting for configuration" + return delivered.is_set() + + def polls() -> list[dict[str, Any]]: + assert starting.wait(timeout=5), "Provider startup was never triggered" + poll = _poll(content="new" if delivered.is_set() else "", acknowledged=delivered.is_set()) + poll["body"]["client"]["capabilities"] = _request( + FLAG_CONFIGURATION_MASK | (DEPENDENT_FLAG_MASK if supported else 0), byte_array=False + )["body"]["client"]["capabilities"] + observed.append(poll) + return list(observed) + + def configure(*, path: str, payload: dict[str, Any]) -> None: + assert path == RC_PATH + assert payload == {"flags": {}} + assert len(observed) >= 3, "Configuration delivered before observing capabilities" + delivered.set() + + library.ffe_start.side_effect = start + agent.rc_requests.side_effect = polls + agent.set_remote_config.side_effect = configure + start_and_deliver(agent, library, config_id="flags", payload={"flags": {}}, supported=supported) + assert delivered.is_set() + assert library.ffe_start.call_count == (1 if blocking else 2) + agent.clear.assert_not_called() diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index c0dd271e277..75829e17da4 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -833,7 +833,9 @@ def ffe_start(self, configuration: dict | None = None) -> bool: if configuration is not None: payload["configuration"] = configuration - resp = self._session.post(self._url("/ffe/start"), json=payload) + # Cold-start tests initialize concurrently with RC delivery. Bound the + # request so a missing configuration cannot leave startup waiting forever. + resp = self._session.post(self._url("/ffe/start"), json=payload, timeout=30) return HTTPStatus(resp.status_code).is_success def ffe_evaluate( From 08dda62e1405eaf475777f8ddc0e9e5008a93db7 Mon Sep 17 00:00:00 2001 From: "helen.han" Date: Thu, 5 Nov 2026 22:16:04 +0000 Subject: [PATCH 5/5] test(ffe): keep RC helpers in the supported utils module Move the RC transport helpers into the existing FFE utils.py and update imports and mock targets so the test discovery convention check passes. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 --- .../parametric/test_ffe/dependent_flags_rc.py | 87 ------------------- .../test_ffe/test_dependent_flags_rc.py | 4 +- tests/parametric/test_ffe/utils.py | 84 +++++++++++++++++- .../test_the_test/test_ffe_rc_capabilities.py | 9 +- 4 files changed, 89 insertions(+), 95 deletions(-) delete mode 100644 tests/parametric/test_ffe/dependent_flags_rc.py diff --git a/tests/parametric/test_ffe/dependent_flags_rc.py b/tests/parametric/test_ffe/dependent_flags_rc.py deleted file mode 100644 index dda4db4c5a1..00000000000 --- a/tests/parametric/test_ffe/dependent_flags_rc.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Capability-gated RC delivery, including cold provider startup and content updates.""" - -from concurrent.futures import ThreadPoolExecutor -import time -from typing import Any - -from tests.parametric.test_ffe.utils import assert_dependent_flag_capability -from utils.dd_constants import RemoteConfigApplyState -from utils.docker_fixtures import ParametricTestClientApi, TestAgentAPI - -RC_PRODUCT = "FFE_FLAGS" - - -def ffe_requests(test_agent: TestAgentAPI) -> list[dict[str, Any]]: - return [ - request for request in test_agent.rc_requests() if RC_PRODUCT in request["body"]["client"].get("products", []) - ] - - -def observe_polls(test_agent: TestAgentAPI, *, supported: bool, minimum: int = 3) -> None: - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - requests = ffe_requests(test_agent) - if requests: - assert_dependent_flag_capability(requests, supported=supported) - if len(requests) >= minimum: - return - time.sleep(0.1) - raise AssertionError(f"Expected at least {minimum} FFE_FLAGS RC polls, got {len(requests)}") - - -def deliver(test_agent: TestAgentAPI, *, config_id: str, payload: dict[str, Any]) -> None: - path = f"datadog/2/{RC_PRODUCT}/{config_id}/config" - previous_requests = test_agent.rc_requests() - previous_target = next( - ( - target - for request in reversed(previous_requests) - for target in request["body"].get("cached_target_files", []) - if target["path"] == path - ), - None, - ) - test_agent.set_remote_config(path=path, payload=payload) - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - # A stale ACK for the same ID must not acknowledge new content. Keep all - # requests for capability checks, but require a new target and ACK together. - for request in test_agent.rc_requests()[len(previous_requests) :]: - body = request["body"] - changed = any( - target["path"] == path and target != previous_target for target in body.get("cached_target_files", []) - ) - acknowledged = any( - state["product"] == RC_PRODUCT - and state["id"] == config_id - and state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value - for state in body["client"]["state"].get("config_states", []) - ) - if changed and acknowledged: - return - time.sleep(0.05) - raise AssertionError(f"The SDK did not acknowledge new configuration content for {config_id}") - - -def start_and_deliver( - test_agent: TestAgentAPI, - test_library: ParametricTestClientApi, - *, - config_id: str, - payload: dict[str, Any], - supported: bool, -) -> None: - # .NET waits for configuration inside /ffe/start; Java can return not-ready - # before delivery. Trigger startup without requiring readiness until after RC. - with ThreadPoolExecutor(max_workers=1) as executor: - started = executor.submit(test_library.ffe_start) - observe_polls(test_agent, supported=supported) - deliver(test_agent, config_id=config_id, payload=payload) - if started.result(timeout=10): - return - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - if test_library.ffe_start(): - return - time.sleep(0.1) - raise AssertionError("Failed to initialize the provider after RC delivery") diff --git a/tests/parametric/test_ffe/test_dependent_flags_rc.py b/tests/parametric/test_ffe/test_dependent_flags_rc.py index 46d9c508786..c96ddc3bb46 100644 --- a/tests/parametric/test_ffe/test_dependent_flags_rc.py +++ b/tests/parametric/test_ffe/test_dependent_flags_rc.py @@ -8,13 +8,13 @@ from typing import Any from tests.parametric.conftest import APMLibrary -from tests.parametric.test_ffe.dependent_flags_rc import ( +from tests.parametric.test_ffe.utils import ( + assert_dependent_flag_capability, deliver, ffe_requests, observe_polls, start_and_deliver, ) -from tests.parametric.test_ffe.utils import assert_dependent_flag_capability from utils import features, pytest, scenarios from utils.docker_fixtures import TestAgentAPI diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index dcd52de58ac..41cb6097861 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -10,14 +10,17 @@ """ from collections.abc import Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor import base64 import gzip import hashlib import json +import time from pathlib import Path from typing import Any -from utils.dd_constants import Capabilities +from utils.dd_constants import Capabilities, RemoteConfigApplyState +from utils.docker_fixtures import ParametricTestClientApi, TestAgentAPI # Varint encoding constants VARINT_DATA_BITS = 0x7F # Lower 7 bits contain data @@ -190,3 +193,82 @@ def assert_dependent_flag_capability(requests: list[dict[str, Any]], *, supporte assert bool(mask & (1 << capability)) is supported, ( f"FFE RC client {client.get('id')} must advertise bit {capability}={supported}; got {mask:#x}" ) + + +RC_PRODUCT = "FFE_FLAGS" + + +def ffe_requests(test_agent: TestAgentAPI) -> list[dict[str, Any]]: + return [ + request for request in test_agent.rc_requests() if RC_PRODUCT in request["body"]["client"].get("products", []) + ] + + +def observe_polls(test_agent: TestAgentAPI, *, supported: bool, minimum: int = 3) -> None: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + requests = ffe_requests(test_agent) + if requests: + assert_dependent_flag_capability(requests, supported=supported) + if len(requests) >= minimum: + return + time.sleep(0.1) + raise AssertionError(f"Expected at least {minimum} FFE_FLAGS RC polls, got {len(requests)}") + + +def deliver(test_agent: TestAgentAPI, *, config_id: str, payload: dict[str, Any]) -> None: + path = f"datadog/2/{RC_PRODUCT}/{config_id}/config" + previous_requests = test_agent.rc_requests() + previous_target = next( + ( + target + for request in reversed(previous_requests) + for target in request["body"].get("cached_target_files", []) + if target["path"] == path + ), + None, + ) + test_agent.set_remote_config(path=path, payload=payload) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + # A stale ACK for the same ID must not acknowledge new content. Keep all + # requests for capability checks, but require a new target and ACK together. + for request in test_agent.rc_requests()[len(previous_requests) :]: + body = request["body"] + changed = any( + target["path"] == path and target != previous_target for target in body.get("cached_target_files", []) + ) + acknowledged = any( + state["product"] == RC_PRODUCT + and state["id"] == config_id + and state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value + for state in body["client"]["state"].get("config_states", []) + ) + if changed and acknowledged: + return + time.sleep(0.05) + raise AssertionError(f"The SDK did not acknowledge new configuration content for {config_id}") + + +def start_and_deliver( + test_agent: TestAgentAPI, + test_library: ParametricTestClientApi, + *, + config_id: str, + payload: dict[str, Any], + supported: bool, +) -> None: + # .NET waits for configuration inside /ffe/start; Java can return not-ready + # before delivery. Trigger startup without requiring readiness until after RC. + with ThreadPoolExecutor(max_workers=1) as executor: + started = executor.submit(test_library.ffe_start) + observe_polls(test_agent, supported=supported) + deliver(test_agent, config_id=config_id, payload=payload) + if started.result(timeout=10): + return + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if test_library.ffe_start(): + return + time.sleep(0.1) + raise AssertionError("Failed to initialize the provider after RC delivery") diff --git a/tests/test_the_test/test_ffe_rc_capabilities.py b/tests/test_the_test/test_ffe_rc_capabilities.py index b99bc12c5b9..c21d93e3855 100644 --- a/tests/test_the_test/test_ffe_rc_capabilities.py +++ b/tests/test_the_test/test_ffe_rc_capabilities.py @@ -5,8 +5,7 @@ from typing import Any from unittest.mock import MagicMock, patch -from tests.parametric.test_ffe.dependent_flags_rc import deliver, ffe_requests, start_and_deliver -from tests.parametric.test_ffe.utils import assert_dependent_flag_capability +from tests.parametric.test_ffe.utils import assert_dependent_flag_capability, deliver, ffe_requests, start_and_deliver from utils import features, pytest, scenarios from utils.dd_constants import Capabilities, RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI, ParametricTestClientApi @@ -106,7 +105,7 @@ def test_update_requires_new_content_and_ack_in_same_poll(self) -> None: [old, old, new_without_ack, wrong_product], [old, old, new_without_ack, wrong_product, new_ack], ] - with patch("tests.parametric.test_ffe.dependent_flags_rc.time.sleep"): + with patch("tests.parametric.test_ffe.utils.time.sleep"): deliver(agent, config_id="flags", payload={"flags": {}}) assert agent.rc_requests.call_count == 5 agent.clear.assert_not_called() @@ -116,8 +115,8 @@ def test_stale_ack_times_out(self) -> None: old = _poll(content="old", acknowledged=True) agent.rc_requests.side_effect = [[old], [old, old]] with ( - patch("tests.parametric.test_ffe.dependent_flags_rc.time.monotonic", side_effect=[0, 0, 11]), - patch("tests.parametric.test_ffe.dependent_flags_rc.time.sleep"), + patch("tests.parametric.test_ffe.utils.time.monotonic", side_effect=[0, 0, 11]), + patch("tests.parametric.test_ffe.utils.time.sleep"), pytest.raises(AssertionError, match="did not acknowledge new configuration content"), ): deliver(agent, config_id="flags", payload={"flags": {}})