From 0a2c969c559c5701dedb3d25d13cacda018a84f4 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 27 Oct 2026 02:26:05 +0000 Subject: [PATCH] test(ffe): cover Ruby agentless EVP routing for both signals --- .github/workflows/run-end-to-end.yml | 28 +++ manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 43 +++- tests/ffe/test_evp_fallback.py | 186 ++++++++++++++++++ utils/_context/_scenarios/__init__.py | 12 ++ .../_context/_scenarios/agentless_endtoend.py | 46 ++++- utils/build/docker/ruby/rails72/app.sh | 6 +- .../build/docker/ruby/rails72/config/puma.rb | 13 ++ utils/mocked_backend/_backend.py | 2 + utils/mocked_backend/_evp.py | 41 ++++ 14 files changed, 374 insertions(+), 8 deletions(-) create mode 100644 tests/ffe/test_evp_fallback.py create mode 100644 utils/mocked_backend/_evp.py diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index a9b7c4a8bf4..c1b16d90f48 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -317,6 +317,34 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS + - name: Run FEATURE_FLAGGING_AGENTLESS_AGENT scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AGENTLESS_AGENT"') + run: ./run.sh FEATURE_FLAGGING_AGENTLESS_AGENT + - name: Run FEATURE_FLAGGING_EVP_404_EXPOSURES scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_404_EXPOSURES"') + run: ./run.sh FEATURE_FLAGGING_EVP_404_EXPOSURES + - name: Run FEATURE_FLAGGING_EVP_404_FLAGEVALUATION scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_404_FLAGEVALUATION"') + run: ./run.sh FEATURE_FLAGGING_EVP_404_FLAGEVALUATION + - name: Run FEATURE_FLAGGING_EVP_405_EXPOSURES scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_405_EXPOSURES"') + run: ./run.sh FEATURE_FLAGGING_EVP_405_EXPOSURES + - name: Run FEATURE_FLAGGING_EVP_405_FLAGEVALUATION scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_405_FLAGEVALUATION"') + run: ./run.sh FEATURE_FLAGGING_EVP_405_FLAGEVALUATION + - name: Run FEATURE_FLAGGING_EVP_403_EXPOSURES scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_403_EXPOSURES"') + run: ./run.sh FEATURE_FLAGGING_EVP_403_EXPOSURES + - name: Run FEATURE_FLAGGING_EVP_403_FLAGEVALUATION scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_403_FLAGEVALUATION"') + run: ./run.sh FEATURE_FLAGGING_EVP_403_FLAGEVALUATION + - name: Run FEATURE_FLAGGING_EVP_AMBIGUOUS_EXPOSURES scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_AMBIGUOUS_EXPOSURES"') + run: ./run.sh FEATURE_FLAGGING_EVP_AMBIGUOUS_EXPOSURES + - name: Run FEATURE_FLAGGING_EVP_AMBIGUOUS_FLAGEVALUATION scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVP_AMBIGUOUS_FLAGEVALUATION"') + run: ./run.sh FEATURE_FLAGGING_EVP_AMBIGUOUS_FLAGEVALUATION + - name: Run AGENT_SUPPORTING_SPAN_EVENTS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"AGENT_SUPPORTING_SPAN_EVENTS"') run: ./run.sh AGENT_SUPPORTING_SPAN_EVENTS diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 7dcdabd0c31..517ddc832cc 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -827,6 +827,7 @@ manifest: excluded_weblog: [poc, uds] tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_evaluation_telemetry_opt_out.py::Test_FFE_Evaluation_Telemetry_Opt_Out_EVP: missing_feature (Requires a release containing dd-trace-dotnet#9351; FFL-2446) + tests/ffe/test_evp_fallback.py: missing_feature (Fault receiver contracts enabled for Ruby only pending SDK validation) tests/ffe/test_exposure_egress.py: v3.36.0 tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: missing_feature (Not yet implemented) diff --git a/manifests/golang.yml b/manifests/golang.yml index 565e95b90ec..7379fa326db 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1732,6 +1732,7 @@ manifest: - declaration: bug (FFL-2182) excluded_weblog: [echo, gin, chi, net-http-orchestrion, net-http, uds-echo] tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) + tests/ffe/test_evp_fallback.py: missing_feature (Fault receiver contracts enabled for Ruby only pending SDK validation) tests/ffe/test_exposure_egress.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: v2.12.0-dev tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: diff --git a/manifests/java.yml b/manifests/java.yml index 0efcd7fa6be..6235be3c93a 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3483,6 +3483,7 @@ manifest: - declaration: bug (FFL-2184) excluded_weblog: [spring-boot] tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) + tests/ffe/test_evp_fallback.py: missing_feature (Fault receiver contracts enabled for Ruby only pending SDK validation) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 1989ba807fc..9ea3067fdad 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1879,6 +1879,7 @@ manifest: - declaration: bug (FFL-2182) excluded_weblog: [express5] tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) + tests/ffe/test_evp_fallback.py: missing_feature (Fault receiver contracts enabled for Ruby only pending SDK validation) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/python.yml b/manifests/python.yml index 92c82db826e..16588a0663b 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2244,6 +2244,7 @@ manifest: tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) + tests/ffe/test_evp_fallback.py: missing_feature (Fault receiver contracts enabled for Ruby only pending SDK validation) tests/ffe/test_exposure_egress.py: v4.2.0-dev tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: - weblog_declaration: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index eafac1fc17b..7591d4dd4c3 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2141,13 +2141,26 @@ manifest: "*": irrelevant rails72: v2.23.0-dev tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) + tests/ffe/test_evp_fallback.py: + - weblog_declaration: + "*": irrelevant + rails72: v2.48.0-dev tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": irrelevant rails72: v2.23.0-dev - tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) - tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: missing_feature (Not yet implemented) - tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: missing_feature (Not yet implemented) + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: + - weblog_declaration: + "*": irrelevant + rails72: v2.48.0-dev + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: + - weblog_declaration: + "*": irrelevant + rails72: v2.48.0-dev + tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Sidecar: + - weblog_declaration: + "*": irrelevant + rails72: v2.48.0-dev tests/ffe/test_exposures_datadog_agent.py: - weblog_declaration: "*": irrelevant @@ -2156,7 +2169,29 @@ manifest: tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Caching_Serial_Id_Cycle: missing_feature (EX-3417) tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Caching_Serial_Id_Disappears: missing_feature (EX-3417) tests/ffe/test_exposures_datadog_agent.py::Test_FFE_Exposure_Serial_Id: missing_feature (EX-3417) - tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Basic: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Context_Bounds: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Count: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Egress_Agentless_Direct: + - weblog_declaration: + "*": irrelevant + rails72: v2.48.0-dev + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Egress_Agentless_Sidecar: + - weblog_declaration: + "*": irrelevant + rails72: v2.48.0-dev + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Egress_Datadog_Agent: + - weblog_declaration: + "*": irrelevant + rails72: v2.48.0-dev + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Load_Aggregation: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Runtime_Default: missing_feature (FFL-2446) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant diff --git a/tests/ffe/test_evp_fallback.py b/tests/ffe/test_evp_fallback.py new file mode 100644 index 00000000000..4e5a10e05b6 --- /dev/null +++ b/tests/ffe/test_evp_fallback.py @@ -0,0 +1,186 @@ +"""Fault-injection contracts exercised by the installed SDK's two production writers.""" + +import hashlib +import json +from typing import Any + +from tests.ffe.utils.evp import EVP_ORIGINS, assert_agentless_evp_intake_request, register_expected_evp_capture +from utils import context, features, interfaces, scenario_crash, scenarios, weblog +from utils._context._scenarios.agentless_endtoend import FeatureFlaggingEVPFallback, FeatureFlaggingAgentlessAgent + + +def events_for(data: dict[str, Any], subject: str) -> list[dict[str, Any]]: + content = data.get("request", {}).get("content", {}) + if not isinstance(content, dict): + return [] + targeting_keys = {subject, f"sha256_{hashlib.sha256(subject.encode()).hexdigest()}"} + return [ + event + for event in content.get("exposures", []) + content.get("flagEvaluations", []) + if event.get("subject", {}).get("id") == subject or event.get("targeting_key") in targeting_keys + ] + + +@scenarios.feature_flagging_evp_404_exposures +@scenarios.feature_flagging_evp_404_flagevaluation +@scenarios.feature_flagging_evp_405_exposures +@scenarios.feature_flagging_evp_405_flagevaluation +@scenarios.feature_flagging_evp_403_exposures +@scenarios.feature_flagging_evp_403_flagevaluation +@scenarios.feature_flagging_evp_ambiguous_exposures +@scenarios.feature_flagging_evp_ambiguous_flagevaluation +@features.feature_flags_exposures +@scenario_crash +class Test_FFE_EVP_Fallback: + def setup_delivery(self) -> None: + scenario = context.scenario + assert isinstance(scenario, FeatureFlaggingEVPFallback) + self.responses = [] + for subject in ("evp-first", "evp-future"): + self.responses.append( + weblog.post( + "/ffe", + json={ + "flag": "empty-targeting-key-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": subject, + "attributes": {}, + }, + ) + ) + if subject == "evp-first": + local_path = f"{scenario.receiver_prefix}/evp_proxy/v4/api/v2/{scenario.fault_signal}" + + def first_captured(data: dict[str, Any], expected_path: str = local_path) -> bool: + return data.get("path") == expected_path and bool(events_for(data, "evp-first")) + + assert interfaces.backend_v2.wait_for(first_captured, timeout=30) + # Wait for the response to be processed before producing the future batch. + interfaces.datadog_direct.wait(3) + for signal in ("exposures", "flagevaluation"): + register_expected_evp_capture(f"/api/v2/{signal}") + + def future_captured(data: dict[str, Any], expected_signal: str = signal) -> bool: + return data.get("path") == f"/api/v2/{expected_signal}" and bool(events_for(data, "evp-future")) + + assert interfaces.datadog_direct.wait_for(future_captured, timeout=35) + + def test_delivery(self) -> None: + scenario = context.scenario + assert isinstance(scenario, FeatureFlaggingEVPFallback) + assert all(response.status_code == 200 for response in self.responses) + assert all(json.loads(response.text)["value"] == "on-value" for response in self.responses) + local = [ + data + for data in interfaces.backend_v2.get_data() + if data.get("path", "").startswith(scenario.receiver_prefix + "/evp_proxy/") + ] + direct = [ + data + for data in interfaces.datadog_direct.get_data() + if data.get("path") in ("/api/v2/exposures", "/api/v2/flagevaluation") + ] + discovery = [ + data for data in interfaces.backend_v2.get_data() if data.get("path") == scenario.receiver_prefix + "/info" + ] + assert len(discovery) == 1 + for data in local + discovery: + headers = {key.lower(): value for key, value in data["request"]["headers"]} + assert "dd-api-key" not in headers + assert "x-datadog-trace-id" not in headers + assert "traceparent" not in headers + assert headers["dd-evp-origin"] == EVP_ORIGINS[context.library.name] + assert headers["dd-evp-origin-version"] == context.library.raw_version + if data in local: + assert headers["x-datadog-evp-subdomain"] == "event-platform-intake" + for data in direct: + assert_agentless_evp_intake_request( + data, + route="direct", + path=data["path"], + expected_api_key=scenario.dd_api_key, + library_name=context.library.name, + library_version=context.library.raw_version, + ) + for signal in ("exposures", "flagevaluation"): + future = [data for data in direct if data["path"].endswith(f"/{signal}") and events_for(data, "evp-future")] + assert len(future) == 1 + assert len(events_for(future[0], "evp-future")) == 1 + if signal == "flagevaluation": + assert events_for(future[0], "evp-future")[0]["evaluation_count"] == 1 + assert not any(events_for(data, "evp-future") for data in local) + other_signal = "flagevaluation" if scenario.fault_signal == "exposures" else "exposures" + other_first = [ + data + for data in local + direct + if data["path"].endswith(f"/{other_signal}") and events_for(data, "evp-first") + ] + assert len(other_first) == 1 + assert len(events_for(other_first[0], "evp-first")) == 1 + first_local = [ + data + for data in local + if data["path"].endswith(f"/{scenario.fault_signal}") and events_for(data, "evp-first") + ] + first_direct = [ + data + for data in direct + if data["path"].endswith(f"/{scenario.fault_signal}") and events_for(data, "evp-first") + ] + assert len(first_local) == 1 + if scenario.fault_mode in ("404", "405"): + assert len(first_direct) == 1 + assert first_local[0]["request"]["content"] == first_direct[0]["request"]["content"] + else: + assert first_direct == [] + + +@scenarios.feature_flagging_agentless_agent +@features.feature_flags_exposures +@scenario_crash +class Test_FFE_EVP_Agentless_Agent: + def setup_delivery(self) -> None: + self.response = weblog.post( + "/ffe", + json={ + "flag": "empty-targeting-key-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "evp-agent", + "attributes": {}, + }, + ) + for signal in ("exposures", "flagevaluation"): + register_expected_evp_capture(f"/api/v2/{signal}") + + def test_delivery(self) -> None: + assert self.response.status_code == 200 + assert json.loads(self.response.text)["value"] == "on-value" + assert isinstance(context.scenario, FeatureFlaggingAgentlessAgent) + assert context.scenario.include_agent is True + for signal in ("exposures", "flagevaluation"): + captures = [ + data + for data in interfaces.library.get_data() + if data.get("path") == f"/evp_proxy/v4/api/v2/{signal}" and events_for(data, "evp-agent") + ] + assert len(captures) == 1 + assert len(events_for(captures[0], "evp-agent")) == 1 + headers = {key.lower(): value for key, value in captures[0]["request"]["headers"]} + assert "dd-api-key" not in headers + assert headers["dd-evp-origin"] == EVP_ORIGINS[context.library.name] + assert headers["dd-evp-origin-version"] == context.library.raw_version + assert headers["x-datadog-evp-subdomain"] == "event-platform-intake" + forwarded = [ + data + for data in interfaces.agent.get_data() + if data.get("path") == f"/api/v2/{signal}" and events_for(data, "evp-agent") + ] + assert len(forwarded) == 1 + assert forwarded[0]["request"]["content"] == captures[0]["request"]["content"] + forwarded_headers = {key.lower(): value for key, value in forwarded[0]["request"]["headers"]} + assert forwarded_headers["dd-evp-origin"] == headers["dd-evp-origin"] + assert forwarded_headers["dd-evp-origin-version"] == headers["dd-evp-origin-version"] + assert forwarded_headers["dd-api-key"] in {context.scenario.dd_api_key, "--redacted--"} + assert not any(events_for(data, "evp-agent") for data in interfaces.datadog_direct.get_data()) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index c79e6396e7a..ffbb510bb28 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -111,6 +111,8 @@ ) from .agentless_endtoend import ( FeatureFlaggingAndExperimentationAgentless, + FeatureFlaggingEVPFallback, + FeatureFlaggingAgentlessAgent, FeatureFlaggingAndExperimentationAgentlessDirect, FeatureFlaggingAndExperimentationAgentlessServerless, ) @@ -319,6 +321,16 @@ class _Scenarios: feature_flagging_and_experimentation_agentless = FeatureFlaggingAndExperimentationAgentless() feature_flagging_and_experimentation_agentless_direct = FeatureFlaggingAndExperimentationAgentlessDirect() feature_flagging_and_experimentation_agentless_serverless = FeatureFlaggingAndExperimentationAgentlessServerless() + feature_flagging_agentless_agent = FeatureFlaggingAgentlessAgent() + feature_flagging_evp_404_exposures = FeatureFlaggingEVPFallback("404", "exposures") + feature_flagging_evp_404_flagevaluation = FeatureFlaggingEVPFallback("404", "flagevaluation") + feature_flagging_evp_405_exposures = FeatureFlaggingEVPFallback("405", "exposures") + feature_flagging_evp_405_flagevaluation = FeatureFlaggingEVPFallback("405", "flagevaluation") + feature_flagging_evp_403_exposures = FeatureFlaggingEVPFallback("403", "exposures") + feature_flagging_evp_403_flagevaluation = FeatureFlaggingEVPFallback("403", "flagevaluation") + feature_flagging_evp_ambiguous_exposures = FeatureFlaggingEVPFallback("ambiguous", "exposures") + feature_flagging_evp_ambiguous_flagevaluation = FeatureFlaggingEVPFallback("ambiguous", "flagevaluation") + feature_flagging_evaluation_telemetry_opt_out = FeatureFlaggingEvaluationTelemetryOptOut() feature_flagging_evaluation_telemetry_opt_out_disabled = FeatureFlaggingEvaluationTelemetryOptOutDisabled() remote_config_mocked_backend_asm_features_nocache = RemoteConfigMockedBackendAsmFeaturesNocache() diff --git a/utils/_context/_scenarios/agentless_endtoend.py b/utils/_context/_scenarios/agentless_endtoend.py index 988318be02c..143de8a88c3 100644 --- a/utils/_context/_scenarios/agentless_endtoend.py +++ b/utils/_context/_scenarios/agentless_endtoend.py @@ -75,11 +75,12 @@ def __init__( other_weblog_containers: tuple[type[TestedContainer], ...] = (), scenario_groups: tuple[ScenarioGroup, ...] = (), use_proxy_for_weblog: bool = False, + include_agent: bool = False, ) -> None: super().__init__( name, doc=doc, - include_agent=False, + include_agent=include_agent, library_interface_timeout=0, other_weblog_containers=other_weblog_containers, scenario_groups=[*scenario_groups, all_scenario_groups.agentless], @@ -148,7 +149,7 @@ def __init__( name: str, *, doc: str = "Validate default agentless UFC delivery and evaluation without a Datadog Agent.", - exposure_egress: Literal["sidecar", "direct"] | None = None, + exposure_egress: Literal["sidecar", "direct", "agent"] | None = None, weblog_env: dict[str, str | None] | None = None, ) -> None: self.exposure_egress = exposure_egress @@ -199,6 +200,7 @@ def __init__( other_weblog_containers=other_weblog_containers, scenario_groups=(all_scenario_groups.ffe,), use_proxy_for_weblog=exposure_egress is not None, + include_agent=exposure_egress == "agent", weblog_env=environment, ) @@ -612,7 +614,13 @@ def _wait_for_expected_evp_captures(self, *, is_empty_test_run: bool) -> None: # but their setup methods never run and therefore register no capture expectations. return - interface = interfaces.datadog_sidecar if self.exposure_egress == "sidecar" else interfaces.datadog_direct + interface = ( + interfaces.agent + if self.exposure_egress == "agent" + else interfaces.datadog_sidecar + if self.exposure_egress == "sidecar" + else interfaces.datadog_direct + ) missing_paths: list[str] = [] for path in sorted(self._expected_evp_capture_paths): @@ -688,3 +696,35 @@ def __init__(self) -> None: doc="Validate exposure delivery with agentless UFC and serverless-init.", exposure_egress="sidecar", ) + + +class FeatureFlaggingEVPFallback(FeatureFlaggingAgentlessEndToEndScenario): + def __init__(self, mode: str, signal: str) -> None: + self.fault_mode = mode + self.fault_signal = signal + self.receiver_prefix = f"/evp-test/{mode}/{signal}" + super().__init__( + f"FEATURE_FLAGGING_EVP_{mode.upper()}_{signal.upper()}", + doc=f"Validate {signal} EVP fallback after local {mode}, with both writers active.", + exposure_egress="direct", + ) + self.weblog_infra.library_container.environment["DD_TRACE_AGENT_URL"] = ( + self.mocked_backend_container.container_url + self.receiver_prefix + ) + + +class FeatureFlaggingAgentlessAgent(FeatureFlaggingAgentlessEndToEndScenario): + def __init__(self) -> None: + super().__init__( + "FEATURE_FLAGGING_AGENTLESS_AGENT", + doc="Validate both agentless Feature Flags signals through a full Datadog Agent.", + exposure_egress="agent", + ) + + # Other Agent traffic uses the HTTP mocked backend; EVP always builds an HTTPS + # intake URL, so capture that traffic separately through the Agent proxy port. + self.agent_container.environment |= { + "DD_EVP_PROXY_CONFIG_DD_URL": "mock-intake.invalid", + "DD_PROXY_HTTPS": f"http://proxy:{ProxyPorts.agent}", + } + self.agent_container.depends_on.append(self.proxy_container) diff --git a/utils/build/docker/ruby/rails72/app.sh b/utils/build/docker/ruby/rails72/app.sh index 5432d180f6e..cd81c14b1a2 100755 --- a/utils/build/docker/ruby/rails72/app.sh +++ b/utils/build/docker/ruby/rails72/app.sh @@ -1,3 +1,7 @@ #!/bin/bash -bundle exec puma -b tcp://0.0.0.0 -p 7777 -w 1 +if [[ "${SYSTEM_TESTS_FFE_SHUTDOWN_FLUSH_ENABLED:-false}" == "true" ]]; then + exec bundle exec puma -b tcp://0.0.0.0 -p 7777 -w 0 +fi + +exec bundle exec puma -b tcp://0.0.0.0 -p 7777 -w 1 diff --git a/utils/build/docker/ruby/rails72/config/puma.rb b/utils/build/docker/ruby/rails72/config/puma.rb index edd1d37efed..aa5369bd327 100644 --- a/utils/build/docker/ruby/rails72/config/puma.rb +++ b/utils/build/docker/ruby/rails72/config/puma.rb @@ -41,3 +41,16 @@ # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart + +if ENV['SYSTEM_TESTS_FFE_SHUTDOWN_FLUSH_ENABLED'] == 'true' + raise_exception_on_sigterm false + require 'json' + require 'time' + on_booted do + # Register after the SDK: Ruby runs this after Puma closes, before the SDK's at_exit flush. + at_exit do + puts JSON.generate(event: 'system_tests.ffe.shutdown.server_closed', timestamp: Time.now.utc.iso8601(6)) + $stdout.flush + end + end +end diff --git a/utils/mocked_backend/_backend.py b/utils/mocked_backend/_backend.py index e90b8ab5c50..172e9e5f606 100644 --- a/utils/mocked_backend/_backend.py +++ b/utils/mocked_backend/_backend.py @@ -14,6 +14,7 @@ from ._request_handler import MockedBackendServerCore, MockBackendV2RequestHandler, HttpMethod, RouteHandler from ._backend_ffe import MockedBackendServerFfeMixin +from ._evp import configure_evp_receiver from ._client import MockedBackendClient from ._utils import CONTROL_URL_PREFIX, HOST_DOCKER_INTERNAL, logger @@ -53,6 +54,7 @@ def __init__( self.add_handler("GET", f"{CONTROL_URL_PREFIX}/health", self._handle_health) self._ffe_configure() + configure_evp_receiver(self) self._thread = threading.Thread(target=self.serve_forever, name=self.thread_name, daemon=True) self._thread.start() diff --git a/utils/mocked_backend/_evp.py b/utils/mocked_backend/_evp.py new file mode 100644 index 00000000000..31751e99d29 --- /dev/null +++ b/utils/mocked_backend/_evp.py @@ -0,0 +1,41 @@ +"""Local EVP fault receiver. Captures happen before rejection or connection loss.""" + +from collections.abc import Mapping +from http import HTTPStatus +import socket +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ._request_handler import MockBackendV2RequestHandler, MockedBackendServerCore + + +def configure_evp_receiver(server: "MockedBackendServerCore") -> None: + for mode in ("404", "405", "403", "ambiguous"): + for signal in ("exposures", "flagevaluation"): + prefix = f"/evp-test/{mode}/{signal}" + server.add_handler("GET", f"{prefix}/info", _info) + for endpoint in ("exposures", "flagevaluation"): + server.add_handler("POST", f"{prefix}/evp_proxy/v4/api/v2/{endpoint}", _delivery) + + +def _info(request: "MockBackendV2RequestHandler", _: bytes | dict | None) -> tuple[Mapping[str, Any], bytes]: + return request.write_json( + HTTPStatus.OK, + { + "endpoints": ["/evp_proxy/v2", "/evp_proxy/v4"], + "evp_proxy_allowed_headers": ["dd-evp-origin", "Dd-Evp-Origin-Version"], + }, + ) + + +def _delivery(request: "MockBackendV2RequestHandler", _: bytes | dict | None) -> tuple[Mapping[str, Any], bytes]: + parts = request.path.split("/") + mode, target = parts[2:4] + if request.path.endswith(f"/{target}"): + if mode == "ambiguous": + # The entire request body has arrived; the sender cannot know whether it was accepted. + request.close_connection = True + request.connection.shutdown(socket.SHUT_RDWR) + return {}, b"" + return request.write_json(HTTPStatus(int(mode)), {}) + return request.write_json(HTTPStatus.ACCEPTED, {})