Skip to content

Commit 5c81661

Browse files
vjfridgeCodex GPT-6
andcommitted
fix: flush feature flag telemetry at invocation end
Use the tracer flush contract within the remaining Lambda budget while preserving handler outcomes and warm providers. Environment: Datadog workspace Co-Authored-By: Codex GPT-6 <noreply@localhost>
1 parent afc1c5f commit 5c81661

7 files changed

Lines changed: 358 additions & 7 deletions

File tree

‎README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,27 @@ Unless explicitly stated otherwise all files in this repository are licensed und
7676
This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2019 Datadog, Inc.
7777

7878
[1]: https://www.datadoghq.com/blog/troubleshoot-lambda-function-request-response-payloads/
79+
80+
## Feature flag telemetry at invocation end
81+
82+
With a tracer and Lambda layer that support automatic feature flag flushing,
83+
the wrapper attempts to send pending evaluation and exposure events before
84+
returning a response or rethrowing a handler exception. No additional handler
85+
code or provider shutdown is required; providers and polling remain available
86+
for warm invocations.
87+
88+
`DD_FEATURE_FLAG_FLUSH_TIMEOUT_MS` sets the maximum added wait in milliseconds
89+
(default: `500`; `0` disables waiting). The actual budget is the smaller of that
90+
cap and the invocation's remaining time minus a 100 ms safety margin. The wrapper
91+
skips waiting when that budget is exhausted or unavailable. Incomplete flushes
92+
are reported in debug logs without changing the handler's result.
93+
94+
Delivery is best effort. A timeout stops waiting but does not cancel or retry
95+
in-flight requests; they may complete after a warm thaw. Abrupt termination can
96+
still lose buffered events. Endpoint acceptance does not guarantee immediate
97+
visibility in the UI. Older tracers safely skip the integration.
98+
99+
Release status: this capability requires the tracer's internal flush contract
100+
version 1. The first supported tracer/library/layer combination is pending
101+
coordinated release and deployed validation; an older published layer does not
102+
provide this behavior merely by setting the timeout variable.

‎datadog_lambda/config.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ def _resolve_env(self, key, default=None, cast=None, depends_on_tracing=False):
8585
enhanced_metrics_enabled = _get_env("DD_ENHANCED_METRICS", "true", as_bool)
8686

8787
flush_in_thread = _get_env("DD_FLUSH_IN_THREAD", "false", as_bool)
88+
feature_flag_flush_timeout_ms = _get_env(
89+
"DD_FEATURE_FLAG_FLUSH_TIMEOUT_MS", "500", float
90+
)
8891
flush_to_log = _get_env("DD_FLUSH_TO_LOG", "false", as_bool)
8992
logs_injection = _get_env("DD_LOGS_INJECTION", "true", as_bool)
9093
merge_xray_traces = _get_env("DD_MERGE_XRAY_TRACES", "false", as_bool)

‎datadog_lambda/feature_flags.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
"""Compatibility boundary for invocation-end feature flag telemetry."""
2+
3+
import logging
4+
import math
5+
6+
from datadog_lambda.config import config
7+
8+
try:
9+
from ddtrace.internal.openfeature import _telemetry
10+
except ImportError:
11+
_telemetry = None
12+
13+
14+
logger = logging.getLogger(__name__)
15+
SAFETY_MARGIN_MS = 100.0
16+
17+
18+
def flush_feature_flag_telemetry(context):
19+
try:
20+
if _telemetry is None or getattr(_telemetry, "FLUSH_API_VERSION", None) != 1:
21+
return
22+
cap_ms = config.feature_flag_flush_timeout_ms
23+
if not math.isfinite(cap_ms) or cap_ms <= 0:
24+
return
25+
# No remaining-time information means no safe invocation budget.
26+
remaining_ms = context.get_remaining_time_in_millis()
27+
budget_ms = min(cap_ms, remaining_ms - SAFETY_MARGIN_MS)
28+
if budget_ms <= 0:
29+
logger.debug(
30+
"Skipping feature flag telemetry flush: no invocation time remains"
31+
)
32+
return
33+
outcome = _telemetry.flush(budget_ms / 1000.0)
34+
if outcome != "completed":
35+
logger.debug("Feature flag telemetry flush incomplete: %s", outcome)
36+
except Exception:
37+
# Never replace the response or the original handler exception.
38+
logger.debug("Feature flag telemetry flush failed", exc_info=True)

‎datadog_lambda/wrapper.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from importlib import import_module
1010
from time import time_ns
1111

12+
from datadog_lambda.feature_flags import flush_feature_flag_telemetry
1213
from datadog_lambda.extension import should_use_extension, flush_extension
1314
from datadog_lambda.cold_start import (
1415
set_cold_start,
@@ -191,18 +192,27 @@ def __call__(self, event, context, **kwargs):
191192
return self.blocking_response
192193
self.response = self.func(event, context, **kwargs)
193194
except Exception:
194-
from datadog_lambda.metric import submit_errors_metric
195-
196-
submit_errors_metric(context)
197-
198-
if self.span:
199-
self.span.set_traceback()
195+
try:
196+
from datadog_lambda.metric import submit_errors_metric
197+
198+
submit_errors_metric(context)
199+
if self.span:
200+
self.span.set_traceback()
201+
except Exception as e:
202+
logger.error(format_err_with_traceback(e))
200203
raise
201204
except BaseException as e:
202205
if "BlockingException" in type(e).__name__:
203206
self.blocking_response = get_asm_blocked_response(self.event_source)
207+
else:
208+
raise
204209
finally:
205-
self._after(event, context)
210+
try:
211+
self._after(event, context)
212+
except Exception as e:
213+
logger.error(format_err_with_traceback(e))
214+
finally:
215+
flush_feature_flag_telemetry(context)
206216
if self.blocking_response:
207217
return self.blocking_response
208218
return self.response
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Feature flag invocation-end smoke test
2+
3+
Status: local wrapper/tracer integration is covered in `tests/test_feature_flags.py`.
4+
The deployed ingestion test has not been run. Do not claim a supported published
5+
layer until the release record below is populated.
6+
7+
## Existing examples and access
8+
9+
The smoke handler initializes its provider and client once at module scope and
10+
reuses them across invocations.
11+
12+
This repository's `scripts/run_integration_tests.sh` documents
13+
`sso-serverless-sandbox-account-admin`. `tests/integration/serverless.yml` uses
14+
`eu-west-1`, a sandbox execution role, and a dedicated deployment bucket. Its
15+
IAM restrictions require names beginning with `integration-tests`. Use that
16+
approved sandbox or obtain an explicitly approved replacement account/role.
17+
Verify that the selected profile can access the approved account before deploying.
18+
19+
## Assemble and deploy
20+
21+
1. Build a Python 3.11 layer containing both candidate changes: the tracer's
22+
internal flush contract version 1 and this Lambda library. The ordinary layer
23+
build installs the declared tracer dependency, so verify the zip actually
24+
contains the candidate tracer instead of a published version without the fix.
25+
2. Package `feature_flags.py` and `openfeature-sdk`. Configure its handler as
26+
`feature_flags.handle`. It uses the ordinary Datadog wrapper and has no flush
27+
or shutdown calls. Reuse the approved sandbox's IAM and deployment settings
28+
from `serverless.yml`, with a unique `integration-tests-*` function name.
29+
3. Configure `FFL_SMOKE_FLAG_KEY` with an existing string test flag that produces
30+
exposure events for the test subjects. Set `DD_SERVICE` to a unique smoke-test
31+
service, `DD_ENV` to the test environment and `DD_FEATURE_FLAG_FLUSH_TIMEOUT_MS=500`.
32+
Keep evaluation telemetry and exposure intake enabled.
33+
4. For direct delivery, use `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless`,
34+
the test organization's `DD_API_KEY`/`DD_SITE`, and no local Agent endpoint.
35+
Supply the API key through the sandbox's approved secret mechanism.
36+
5. Validate Agent delivery separately with a reachable Agent that advertises the
37+
relevant EVP routes. The Lambda extension does not acquire new intake routes
38+
from this change; do not treat extension presence as proof of Agent EVP support.
39+
40+
## Validate
41+
42+
Invoke the same function three times with distinct synthetic subjects, for
43+
example `{"subject":"ffl3412-<run>-1"}`, then a different subject with
44+
`"raise":true`, then a third success. Distinct subjects avoid intentional exposure
45+
deduplication. Confirm the second invocation returns the intentional handler
46+
error, the other two succeed, and logs show the same `sandbox_id` with increasing
47+
invocation counts. A changed sandbox ID means warm reuse was not demonstrated.
48+
49+
Check both evaluation and exposure ingestion for every subject at the configured
50+
Datadog site, including the raising invocation. Capture the invocation request
51+
IDs and ingestion evidence. `evaluation_ms` should be below the background flush
52+
intervals; measure invocation duration separately because cleanup adds wait.
53+
Endpoint acceptance can precede UI visibility. No further invocation or deliberate
54+
sleep in the handler should be needed to trigger delivery.
55+
56+
Repeat with a slow/unavailable endpoint and a short remaining-time budget; verify
57+
the original response/error is retained and the extra caller wait stays within
58+
the configured cap or remaining time minus 100 ms. Inspect debug diagnostics for
59+
incomplete flushes. Remove only the smoke-test stack/resources created by the run.
60+
61+
## Release record
62+
63+
Record the first supported **published** tracer version, Lambda library version,
64+
and layer ARN/version together, plus runtime, architecture, configuration source,
65+
service/environment, request IDs, and ingestion evidence. All are pending deployed
66+
validation and release coordination. Release the tracer first, then the wrapper
67+
and layer that contain it. Older tracer versions safely skip the capability.

‎tests/integration/feature_flags.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
"""Short-invocation smoke handler; use an exposure-enabled string test flag."""
2+
3+
import json
4+
import os
5+
import time
6+
import uuid
7+
8+
from datadog_lambda.wrapper import datadog_lambda_wrapper
9+
from ddtrace.openfeature import DataDogProvider
10+
from openfeature import api
11+
from openfeature.evaluation_context import EvaluationContext
12+
13+
provider = DataDogProvider()
14+
api.set_provider(provider)
15+
client = api.get_client("lambda-flush-smoke")
16+
sandbox_id = uuid.uuid4().hex
17+
invocations = 0
18+
19+
20+
@datadog_lambda_wrapper
21+
def handle(event, context):
22+
global invocations
23+
invocations += 1
24+
started = time.monotonic()
25+
subject = event["subject"]
26+
details = client.get_string_details(
27+
os.environ["FFL_SMOKE_FLAG_KEY"],
28+
"smoke-default",
29+
EvaluationContext(targeting_key=subject),
30+
)
31+
result = {
32+
"request_id": context.aws_request_id,
33+
"sandbox_id": sandbox_id,
34+
"invocation": invocations,
35+
"subject": subject,
36+
"value": details.value,
37+
"variant": details.variant,
38+
"error_code": str(details.error_code) if details.error_code else None,
39+
"evaluation_ms": (time.monotonic() - started) * 1000,
40+
}
41+
# Only synthetic test subjects belong in this smoke-test log.
42+
print(json.dumps(result))
43+
if details.error_code:
44+
raise RuntimeError(
45+
"Smoke flag did not resolve; verify test flag and configuration"
46+
)
47+
if event.get("raise"):
48+
raise RuntimeError("intentional smoke-test handler failure")
49+
return result

‎tests/test_feature_flags.py‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
import importlib.util
2+
import unittest
3+
from unittest.mock import Mock, patch
4+
5+
from datadog_lambda import feature_flags
6+
from datadog_lambda import wrapper
7+
8+
9+
class FeatureFlagFlushTest(unittest.TestCase):
10+
def test_budget(self):
11+
for remaining, cap, expected in [
12+
(5000, 500, 0.5),
13+
(200, 500, 0.1),
14+
(101, 500, 0.001),
15+
]:
16+
with self.subTest(remaining=remaining), patch.object(
17+
feature_flags, "_telemetry"
18+
) as telemetry:
19+
telemetry.FLUSH_API_VERSION = 1
20+
telemetry.flush.return_value = "completed"
21+
with patch.object(
22+
type(feature_flags.config), "feature_flag_flush_timeout_ms", cap
23+
):
24+
feature_flags.flush_feature_flag_telemetry(
25+
Mock(get_remaining_time_in_millis=lambda: remaining)
26+
)
27+
telemetry.flush.assert_called_once_with(expected)
28+
29+
def test_no_budget(self):
30+
for remaining, cap in [
31+
(100, 500),
32+
(0, 500),
33+
(1000, 0),
34+
(1000, -1),
35+
(1000, float("nan")),
36+
]:
37+
with self.subTest(remaining=remaining, cap=cap), patch.object(
38+
feature_flags, "_telemetry"
39+
) as telemetry:
40+
telemetry.FLUSH_API_VERSION = 1
41+
with patch.object(
42+
type(feature_flags.config), "feature_flag_flush_timeout_ms", cap
43+
):
44+
feature_flags.flush_feature_flag_telemetry(
45+
Mock(get_remaining_time_in_millis=lambda: remaining)
46+
)
47+
telemetry.flush.assert_not_called()
48+
49+
def test_old_tracer(self):
50+
for telemetry in [None, Mock(FLUSH_API_VERSION=2)]:
51+
with patch.object(feature_flags, "_telemetry", telemetry):
52+
feature_flags.flush_feature_flag_telemetry(None)
53+
54+
def test_missing_context_and_telemetry_errors_are_safe(self):
55+
with patch.object(feature_flags, "_telemetry") as telemetry:
56+
telemetry.FLUSH_API_VERSION = 1
57+
telemetry.flush.side_effect = RuntimeError("unavailable")
58+
feature_flags.flush_feature_flag_telemetry(None)
59+
feature_flags.flush_feature_flag_telemetry(
60+
Mock(get_remaining_time_in_millis=lambda: 1000)
61+
)
62+
telemetry.flush.assert_called_once()
63+
64+
def test_incomplete_diagnostic(self):
65+
with patch.object(feature_flags, "_telemetry") as telemetry, patch.object(
66+
feature_flags, "logger"
67+
) as log:
68+
telemetry.FLUSH_API_VERSION = 1
69+
telemetry.flush.return_value = "timed_out"
70+
feature_flags.flush_feature_flag_telemetry(
71+
Mock(get_remaining_time_in_millis=lambda: 1000)
72+
)
73+
log.debug.assert_called_with(
74+
"Feature flag telemetry flush incomplete: %s", "timed_out"
75+
)
76+
77+
def test_wrapper_preserves_result_and_exception_when_cleanup_fails(self):
78+
for raises in [False, True]:
79+
original = ValueError("handler error")
80+
handler = (
81+
Mock(side_effect=original)
82+
if raises
83+
else Mock(return_value={"ok": True})
84+
)
85+
# Isolate invocation control flow from tracing/metrics initialization.
86+
decorated = object.__new__(wrapper._LambdaDecorator)
87+
decorated.func = handler
88+
decorated.blocking_response = None
89+
decorated.span = None
90+
decorated._before = Mock()
91+
decorated._after = Mock(side_effect=RuntimeError("cleanup error"))
92+
context = Mock(get_remaining_time_in_millis=lambda: 30000)
93+
with patch.object(wrapper, "flush_feature_flag_telemetry") as flush, patch(
94+
"datadog_lambda.metric.submit_errors_metric",
95+
side_effect=RuntimeError("metrics error"),
96+
):
97+
if raises:
98+
with self.assertRaises(ValueError) as caught:
99+
decorated({}, context)
100+
self.assertIs(caught.exception, original)
101+
else:
102+
self.assertEqual(decorated({}, context), {"ok": True})
103+
flush.assert_called_once_with(context)
104+
105+
106+
class FeatureFlagTracerIntegrationTest(unittest.TestCase):
107+
@unittest.skipUnless(
108+
feature_flags._telemetry is not None
109+
and importlib.util.find_spec("openfeature") is not None,
110+
"requires tracer flush contract v1 and the optional OpenFeature SDK",
111+
)
112+
def test_short_warm_invocations_flush_both_writers(self):
113+
from ddtrace.internal.openfeature._flagevaluation_writer import (
114+
FlagEvaluationWriter,
115+
_EvalEvent,
116+
)
117+
from ddtrace.internal.openfeature._evp_transport import EVPRoute
118+
from ddtrace.internal.openfeature.writer import ExposureWriter
119+
120+
route = EVPRoute("http://agent:8126", "/evp_proxy/v2", {})
121+
selector = Mock(configuration_source="agentless")
122+
selector.select.return_value = route
123+
selector.send.side_effect = lambda route, send: send(route)
124+
connection = Mock()
125+
connection.getresponse.return_value.status = 202
126+
evaluations = FlagEvaluationWriter(
127+
route_selector=selector, connection_factory=lambda *a, **k: connection
128+
)
129+
exposures = ExposureWriter(
130+
enabled=True,
131+
route_selector=selector,
132+
connection_factory=lambda *a, **k: connection,
133+
)
134+
original = ValueError("handler failure")
135+
136+
def handler(event, context):
137+
evaluations.enqueue(
138+
_EvalEvent("flag", "on", "allocation", "subject", {}, False, "", 1)
139+
)
140+
exposures.enqueue({"timestamp": 1})
141+
if event.get("raise"):
142+
raise original
143+
return {"ok": True}
144+
145+
decorated = object.__new__(wrapper._LambdaDecorator)
146+
decorated.func = handler
147+
decorated.blocking_response = None
148+
decorated.span = None
149+
decorated._before = Mock()
150+
decorated._after = Mock(side_effect=RuntimeError("unrelated cleanup"))
151+
context = Mock(get_remaining_time_in_millis=lambda: 30000)
152+
with patch("datadog_lambda.metric.submit_errors_metric"):
153+
self.assertEqual(decorated({}, context), {"ok": True})
154+
self.assertEqual(connection.request.call_count, 2)
155+
with self.assertRaises(ValueError) as caught:
156+
decorated({"raise": True}, context)
157+
self.assertIs(caught.exception, original)
158+
self.assertEqual(connection.request.call_count, 4)
159+
self.assertEqual(decorated({}, context), {"ok": True})
160+
self.assertEqual(connection.request.call_count, 6)

0 commit comments

Comments
 (0)