diff --git a/docs/understand/scenarios/parametric.md b/docs/understand/scenarios/parametric.md index 4a901b56c0b..7ef30ebf0e2 100644 --- a/docs/understand/scenarios/parametric.md +++ b/docs/understand/scenarios/parametric.md @@ -308,3 +308,38 @@ def test_stuff(test_library): [2]: https://docs.pytest.org/en/6.2.x/usage.html#specifying-tests-selecting-tests [3]: https://github.com/roydahan/xunit-viewer [4]: ../../utils/build/docker/cpp/parametric/CMakeLists.txt + +### Dependent flags over Remote Configuration + +`tests/parametric/test_ffe/test_dependent_flags_rc.py` checks the wire contract for +server SDKs using Agent RC. The supported-build test requires bit 50 on every +observed request subscribing to `FFE_FLAGS`, before any dependent configuration +is delivered and after configuration updates. It verifies prerequisite variant +keys rather than flag values, using configuration delivered exclusively by RC. +The test agent models capability-gated delivery; this does not validate production +backend targeting. + +Use the [custom tracer build instructions](../../execute/binaries.md) to select +the SDK and native library under test. With an unreleased supported build, force +the positive class so a missing-feature manifest cannot turn a regression into +an expected failure: + +```sh +TEST_LIBRARY=python ./run.sh PARAMETRIC \ + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC \ + -F tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC +``` + +For a build known to lack dependent-flag support, select and force +`Test_FFE_Without_Dependent_Flags_RC` instead. It must receive and evaluate an +ordinary RC flag while advertising no bit 50. No RC traffic is a failure, not +proof that an unsupported build is safe. Python 4.16.0 is the explicit legacy +baseline in the manifest; other legacy/native combinations can use `-F`. + +Enable the positive class in each SDK's manifest when its dependent-flag support +ships, alongside the evaluation suite. All SDKs retain missing-feature declarations +until a release containing dependent-flag support is confirmed. Python's +implementation is tracked in [dd-trace-py #21346](https://github.com/DataDog/dd-trace-py/pull/21346); +use the forced positive class to test that build before enabling a release threshold. +For Ruby, verify the loaded native package as well as the SDK version. Do not infer +expected support from whether the SDK happens to advertise bit 50. diff --git a/manifests/cpp.yml b/manifests/cpp.yml index bc2ba4b839d..e6dec7de8c9 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -187,6 +187,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_exposure_hooks.py: incomplete_test_app (Exposure hook observations are not implemented) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index dbbad51aee5..bd482146553 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1218,6 +1218,8 @@ manifest: : # TODO: a lower version might be supported - declaration: bug (FFL-3325) component_version: <3.55.0 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script diff --git a/manifests/golang.yml b/manifests/golang.yml index 1bed7c35590..7811357e6d6 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2185,6 +2185,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/java.yml b/manifests/java.yml index 08b1f162174..c2dcb1dfb7e 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4266,6 +4266,8 @@ manifest: ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_stable_true_preserves_default_agentless : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: v1.69.0-SNAPSHOT diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index a22f77adf91..ecaff15ff09 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2700,6 +2700,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: *ref_5_47_0 tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: *ref_5_56_0 tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/php.yml b/manifests/php.yml index a0b173d64df..0890e93235c 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1649,6 +1649,8 @@ manifest: ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_stable_true_preserves_default_agentless : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/python.yml b/manifests/python.yml index 7f14c4e2e6e..74c6ead2037 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -3153,6 +3153,10 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: v2.20.0 tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release; dd-trace-py PR 21346) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: + - declaration: irrelevant (Legacy coverage is pinned to Python 4.16.0) + component_version: '<4.16.0 || >4.16.0' tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index ce005e438f3..cd6c9d5af88 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2711,6 +2711,8 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) diff --git a/manifests/rust.yml b/manifests/rust.yml index 8de4581130c..03926dc6167 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -299,6 +299,8 @@ manifest: tests/parametric/test_dynamic_configuration.py::Test_DynamicConfigSdkConfiguration::test_sdk_config_profiling_enabled: missing_feature (SDK_CONFIGURATION capability not yet implemented) tests/parametric/test_extract_behavior.py: '>=0.4.1-dev' tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Dependent_Flags_RC: missing_feature (Dependent flag support must be enabled with the SDK release) + tests/parametric/test_ffe/test_dependent_flags_rc.py::Test_FFE_Without_Dependent_Flags_RC: irrelevant (Requires a known legacy build without dependent flag support) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_dependent_flags: missing_feature (FFL-1694) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature diff --git a/tests/parametric/test_ffe/test_dependent_flags_rc.py b/tests/parametric/test_ffe/test_dependent_flags_rc.py new file mode 100644 index 00000000000..c96ddc3bb46 --- /dev/null +++ b/tests/parametric/test_ffe/test_dependent_flags_rc.py @@ -0,0 +1,123 @@ +"""Dependent-flag capability and evaluation contract on the Agent RC transport. + +Supported and legacy builds have separate manifest-controlled expectations. Never +infer support from the advertised bit: doing so would hide a missing-bit regression. +""" + +import time +from typing import Any + +from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import ( + assert_dependent_flag_capability, + deliver, + ffe_requests, + observe_polls, + start_and_deliver, +) +from utils import features, pytest, scenarios +from utils.docker_fixtures import TestAgentAPI + +ENV = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", +} + + +def _config(*, parent_variant: str, dependent: bool) -> dict[str, Any]: + parent = { + "key": "parent", + "enabled": True, + "variationType": "STRING", + # The selected variant key deliberately differs from its value. + "variations": { + "approved": {"key": "approved", "value": "not-the-variant-key"}, + "rejected": {"key": "rejected", "value": "approved"}, + }, + "allocations": [ + {"key": "parent-default", "splits": [{"variationKey": parent_variant, "shards": []}], "doLog": False} + ], + } + flags = {"parent": parent} + if dependent: + flags["child"] = { + "key": "child", + "enabled": True, + "variationType": "STRING", + "variations": {key: {"key": key, "value": key} for key in ("matched", "unmatched")}, + "allocations": [ + { + "key": "prerequisite", + "rules": [ + { + "conditions": [ + {"flagEvaluation": {"key": "parent"}, "operator": "ONE_OF", "value": ["approved"]} + ] + } + ], + "splits": [{"variationKey": "matched", "shards": []}], + "doLog": False, + }, + {"key": "fallback", "splits": [{"variationKey": "unmatched", "shards": []}], "doLog": False}, + ], + } + return {"createdAt": "2026-11-03T00:00:00Z", "format": "SERVER", "environment": {"name": "test"}, "flags": flags} + + +def _assert_value(test_library: APMLibrary, flag: str, value: str) -> None: + deadline = time.monotonic() + 5 + while True: + result = test_library.ffe_evaluate( + flag=flag, + variation_type="STRING", + default_value="caller-default", + targeting_key="subject", + ) + if result.get("value") == value and result.get("errorCode") in (None, ""): + return + assert time.monotonic() < deadline, result + time.sleep(0.1) + + +@scenarios.parametric +@features.feature_flags_dynamic_evaluation +class Test_FFE_Dependent_Flags_RC: + @pytest.mark.parametrize("library_env", [ENV]) + def test_capability_and_gated_delivery(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """Advertise before delivery, evaluate variant keys, and retain the bit after updates.""" + start_and_deliver( + test_agent, + test_library, + config_id="dependent-flags", + payload=_config(parent_variant="approved", dependent=True), + supported=True, + ) + _assert_value(test_library, "child", "matched") + previous_count = len(ffe_requests(test_agent)) + observe_polls(test_agent, supported=True, minimum=previous_count + 2) + + assert_dependent_flag_capability(ffe_requests(test_agent), supported=True) + deliver(test_agent, config_id="dependent-flags", payload=_config(parent_variant="rejected", dependent=True)) + _assert_value(test_library, "child", "unmatched") + previous_count = len(ffe_requests(test_agent)) + observe_polls(test_agent, supported=True, minimum=previous_count + 2) + + +@scenarios.parametric +@features.feature_flags_dynamic_evaluation +class Test_FFE_Without_Dependent_Flags_RC: + @pytest.mark.parametrize("library_env", [ENV]) + def test_legacy_build_does_not_advertise(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """A known legacy build receives ordinary flags and never advertises bit 50.""" + start_and_deliver( + test_agent, + test_library, + config_id="ordinary-flags", + payload=_config(parent_variant="approved", dependent=False), + supported=False, + ) + _assert_value(test_library, "parent", "not-the-variant-key") + previous_count = len(ffe_requests(test_agent)) + observe_polls(test_agent, supported=False, minimum=previous_count + 2) diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index b82b1c64c0a..41cb6097861 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -10,13 +10,18 @@ """ from collections.abc import Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor import base64 import gzip import hashlib import json +import time from pathlib import Path from typing import Any +from utils.dd_constants import Capabilities, RemoteConfigApplyState +from utils.docker_fixtures import ParametricTestClientApi, TestAgentAPI + # Varint encoding constants VARINT_DATA_BITS = 0x7F # Lower 7 bits contain data VARINT_CONTINUATION_BIT = 0x80 # MSB indicates more bytes follow @@ -175,3 +180,95 @@ def ffe_events_from_requests(requests: Sequence[Mapping[str, Any]], flag: str) - payload = json.loads(body) events[destination].extend(event for event in payload[field] if event.get("flag", {}).get("key") == flag) return events + + +def assert_dependent_flag_capability(requests: list[dict[str, Any]], *, supported: bool) -> None: + assert requests, "No FFE_FLAGS RC requests observed; absence of traffic is not evidence of compatibility" + capability = Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1 + for request in requests: + client = request["body"]["client"] + raw = client.get("capabilities", "") + decoded = bytes(raw) if isinstance(raw, list) else base64.b64decode(raw) + mask = int.from_bytes(decoded, byteorder="big") + assert bool(mask & (1 << capability)) is supported, ( + f"FFE RC client {client.get('id')} must advertise bit {capability}={supported}; got {mask:#x}" + ) + + +RC_PRODUCT = "FFE_FLAGS" + + +def ffe_requests(test_agent: TestAgentAPI) -> list[dict[str, Any]]: + return [ + request for request in test_agent.rc_requests() if RC_PRODUCT in request["body"]["client"].get("products", []) + ] + + +def observe_polls(test_agent: TestAgentAPI, *, supported: bool, minimum: int = 3) -> None: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + requests = ffe_requests(test_agent) + if requests: + assert_dependent_flag_capability(requests, supported=supported) + if len(requests) >= minimum: + return + time.sleep(0.1) + raise AssertionError(f"Expected at least {minimum} FFE_FLAGS RC polls, got {len(requests)}") + + +def deliver(test_agent: TestAgentAPI, *, config_id: str, payload: dict[str, Any]) -> None: + path = f"datadog/2/{RC_PRODUCT}/{config_id}/config" + previous_requests = test_agent.rc_requests() + previous_target = next( + ( + target + for request in reversed(previous_requests) + for target in request["body"].get("cached_target_files", []) + if target["path"] == path + ), + None, + ) + test_agent.set_remote_config(path=path, payload=payload) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + # A stale ACK for the same ID must not acknowledge new content. Keep all + # requests for capability checks, but require a new target and ACK together. + for request in test_agent.rc_requests()[len(previous_requests) :]: + body = request["body"] + changed = any( + target["path"] == path and target != previous_target for target in body.get("cached_target_files", []) + ) + acknowledged = any( + state["product"] == RC_PRODUCT + and state["id"] == config_id + and state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value + for state in body["client"]["state"].get("config_states", []) + ) + if changed and acknowledged: + return + time.sleep(0.05) + raise AssertionError(f"The SDK did not acknowledge new configuration content for {config_id}") + + +def start_and_deliver( + test_agent: TestAgentAPI, + test_library: ParametricTestClientApi, + *, + config_id: str, + payload: dict[str, Any], + supported: bool, +) -> None: + # .NET waits for configuration inside /ffe/start; Java can return not-ready + # before delivery. Trigger startup without requiring readiness until after RC. + with ThreadPoolExecutor(max_workers=1) as executor: + started = executor.submit(test_library.ffe_start) + observe_polls(test_agent, supported=supported) + deliver(test_agent, config_id=config_id, payload=payload) + if started.result(timeout=10): + return + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if test_library.ffe_start(): + return + time.sleep(0.1) + raise AssertionError("Failed to initialize the provider after RC delivery") diff --git a/tests/test_the_test/test_ffe_rc_capabilities.py b/tests/test_the_test/test_ffe_rc_capabilities.py new file mode 100644 index 00000000000..c21d93e3855 --- /dev/null +++ b/tests/test_the_test/test_ffe_rc_capabilities.py @@ -0,0 +1,160 @@ +"""Regressions for RC capability assertions, gated startup, and update acknowledgement.""" + +import base64 +from threading import Event +from typing import Any +from unittest.mock import MagicMock, patch + +from tests.parametric.test_ffe.utils import assert_dependent_flag_capability, deliver, ffe_requests, start_and_deliver +from utils import features, pytest, scenarios +from utils.dd_constants import Capabilities, RemoteConfigApplyState +from utils.docker_fixtures import TestAgentAPI, ParametricTestClientApi + +DEPENDENT_FLAG_MASK = 1 << Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1 +FLAG_CONFIGURATION_MASK = 1 << Capabilities.FFE_FLAG_CONFIGURATION_RULES + + +def _request(mask: int, *, byte_array: bool) -> dict[str, Any]: + raw = mask.to_bytes(7, byteorder="big") + return { + "body": { + "client": {"id": "ffe-client", "capabilities": list(raw) if byte_array else base64.b64encode(raw).decode()} + } + } + + +@scenarios.test_the_test +@features.not_reported +class Test_FFE_RC_Capability_Assertion: + @pytest.mark.parametrize("byte_array", [False, True]) + def test_supported_and_legacy(self, *, byte_array: bool) -> None: + assert_dependent_flag_capability( + [_request(FLAG_CONFIGURATION_MASK | DEPENDENT_FLAG_MASK, byte_array=byte_array)], supported=True + ) + assert_dependent_flag_capability([_request(FLAG_CONFIGURATION_MASK, byte_array=byte_array)], supported=False) + + @pytest.mark.parametrize("byte_array", [False, True]) + def test_missing_bit_fails_even_if_later_poll_recovers(self, *, byte_array: bool) -> None: + requests = [ + _request(FLAG_CONFIGURATION_MASK, byte_array=byte_array), + _request(FLAG_CONFIGURATION_MASK | DEPENDENT_FLAG_MASK, byte_array=byte_array), + ] + with pytest.raises(AssertionError, match=f"bit {Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1}=True"): + assert_dependent_flag_capability(requests, supported=True) + + def test_false_advertisement_fails(self) -> None: + with pytest.raises(AssertionError, match=f"bit {Capabilities.FFE_FLAG_DEPENDENT_FLAG_VARIANT_KEY_V1}=False"): + assert_dependent_flag_capability([_request(DEPENDENT_FLAG_MASK, byte_array=False)], supported=False) + + def test_no_traffic_is_not_a_pass(self) -> None: + with pytest.raises(AssertionError, match="No FFE_FLAGS RC requests"): + assert_dependent_flag_capability([], supported=False) + + +RC_PATH = "datadog/2/FFE_FLAGS/flags/config" + + +def _poll(*, content: str = "", acknowledged: bool = False, product: str = "FFE_FLAGS") -> dict[str, Any]: + request = _request(FLAG_CONFIGURATION_MASK | DEPENDENT_FLAG_MASK, byte_array=False) + request["method"] = "GET" + request["body"]["client"].update( + products=["FFE_FLAGS"], + state={ + "config_states": [ + { + "id": "flags", + "product": product, + "apply_state": ( + RemoteConfigApplyState.ACKNOWLEDGED.value + if acknowledged + else RemoteConfigApplyState.UNACKNOWLEDGED.value + ), + } + ] + }, + ) + request["body"]["cached_target_files"] = ( + [{"path": RC_PATH, "hashes": [{"algorithm": "sha256", "hash": content}]}] if content else [] + ) + return request + + +@scenarios.test_the_test +@features.not_reported +class Test_FFE_RC_Delivery: + def test_get_and_post_polls_keep_product_filter(self) -> None: + agent = MagicMock(spec=TestAgentAPI) + get = _poll() + post = _poll() + post["method"] = "POST" + unrelated = _poll() + unrelated["body"]["client"]["products"] = ["APM_TRACING"] + agent.rc_requests.side_effect = lambda *, post_only=False: ([post] if post_only else [get, unrelated, post]) + assert ffe_requests(agent) == [get, post] + + def test_update_requires_new_content_and_ack_in_same_poll(self) -> None: + agent = MagicMock(spec=TestAgentAPI) + old = _poll(content="old", acknowledged=True) + new_without_ack = _poll(content="new") + wrong_product = _poll(content="new", acknowledged=True, product="APM_TRACING") + new_ack = _poll(content="new", acknowledged=True) + agent.rc_requests.side_effect = [ + [old], + [old, old], + [old, old, new_without_ack], + [old, old, new_without_ack, wrong_product], + [old, old, new_without_ack, wrong_product, new_ack], + ] + with patch("tests.parametric.test_ffe.utils.time.sleep"): + deliver(agent, config_id="flags", payload={"flags": {}}) + assert agent.rc_requests.call_count == 5 + agent.clear.assert_not_called() + + def test_stale_ack_times_out(self) -> None: + agent = MagicMock(spec=TestAgentAPI) + old = _poll(content="old", acknowledged=True) + agent.rc_requests.side_effect = [[old], [old, old]] + with ( + patch("tests.parametric.test_ffe.utils.time.monotonic", side_effect=[0, 0, 11]), + patch("tests.parametric.test_ffe.utils.time.sleep"), + pytest.raises(AssertionError, match="did not acknowledge new configuration content"), + ): + deliver(agent, config_id="flags", payload={"flags": {}}) + + @pytest.mark.parametrize("blocking", [True, False]) + @pytest.mark.parametrize("supported", [True, False]) + def test_cold_start_delivers_before_requiring_readiness(self, *, blocking: bool, supported: bool) -> None: + agent = MagicMock(spec=TestAgentAPI) + library = MagicMock(spec=ParametricTestClientApi) + starting = Event() + delivered = Event() + observed: list[dict[str, Any]] = [] + + def start() -> bool: + starting.set() + if blocking: + assert delivered.wait(timeout=5), "Startup blocked waiting for configuration" + return delivered.is_set() + + def polls() -> list[dict[str, Any]]: + assert starting.wait(timeout=5), "Provider startup was never triggered" + poll = _poll(content="new" if delivered.is_set() else "", acknowledged=delivered.is_set()) + poll["body"]["client"]["capabilities"] = _request( + FLAG_CONFIGURATION_MASK | (DEPENDENT_FLAG_MASK if supported else 0), byte_array=False + )["body"]["client"]["capabilities"] + observed.append(poll) + return list(observed) + + def configure(*, path: str, payload: dict[str, Any]) -> None: + assert path == RC_PATH + assert payload == {"flags": {}} + assert len(observed) >= 3, "Configuration delivered before observing capabilities" + delivered.set() + + library.ffe_start.side_effect = start + agent.rc_requests.side_effect = polls + agent.set_remote_config.side_effect = configure + start_and_deliver(agent, library, config_id="flags", payload={"flags": {}}, supported=supported) + assert delivered.is_set() + assert library.ffe_start.call_count == (1 if blocking else 2) + agent.clear.assert_not_called() diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index c0dd271e277..75829e17da4 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -833,7 +833,9 @@ def ffe_start(self, configuration: dict | None = None) -> bool: if configuration is not None: payload["configuration"] = configuration - resp = self._session.post(self._url("/ffe/start"), json=payload) + # Cold-start tests initialize concurrently with RC delivery. Bound the + # request so a missing configuration cannot leave startup waiting forever. + resp = self._session.post(self._url("/ffe/start"), json=payload, timeout=30) return HTTPStatus(resp.status_code).is_success def ffe_evaluate(