diff --git a/.github/workflows/update-ffe-fixtures.yml b/.github/workflows/update-ffe-fixtures.yml index 778db406a4b..f41a1dd9d6a 100644 --- a/.github/workflows/update-ffe-fixtures.yml +++ b/.github/workflows/update-ffe-fixtures.yml @@ -77,7 +77,7 @@ jobs: ## Changes - Refresh the checked-in snapshot to [${{ steps.fixtures.outputs.source_commit }}](https://github.com/DataDog/ffe-system-test-data/commit/${{ steps.fixtures.outputs.source_commit }}), containing ${{ steps.fixtures.outputs.fixture_count }} evaluation cases. + Refresh the checked-in snapshot to [${{ steps.fixtures.outputs.source_commit }}](https://github.com/DataDog/ffe-system-test-data/commit/${{ steps.fixtures.outputs.source_commit }}), containing ${{ steps.fixtures.outputs.fixture_count }} evaluation cases and ${{ steps.fixtures.outputs.selection_fixture_count }} selection SDK cases. Ordinary test runs use the snapshot without network access. Resolve newly exposed evaluator failures before merging; correct shared expectations upstream only when they are wrong. Scheduled updates leave an existing update PR untouched to preserve review and fixes. diff --git a/docs/understand/scenarios/parametric.md b/docs/understand/scenarios/parametric.md index 4a901b56c0b..5839c4e13e9 100644 --- a/docs/understand/scenarios/parametric.md +++ b/docs/understand/scenarios/parametric.md @@ -220,14 +220,16 @@ Not all endpoint implementations per language are up to spec with regards to the #### Standalone Rules Configuration -`test_library.ffe_fetch_rules_configuration(options)` calls `POST /ffe/fetch-rules-configuration` (currently implemented in Node.js only). It fetches and serializes configuration using the public standalone FFE helpers in a fresh process, without initializing a tracer or OpenFeature provider. The response is either `{ "wire": "..." }` or `{ "error": { "name": "ConfigurationFetchError", "code": "...", "message": "...", "status": 403 } }`. `status` is present only for HTTP failures. Unexpected failures return HTTP 500. +`test_library.ffe_fetch_rules_configuration(options)` calls `POST /ffe/fetch-rules-configuration` (currently implemented in Node.js only). It fetches and serializes configuration using the public standalone FFE helpers in a fresh process, without initializing a tracer or OpenFeature provider. The response is either `{ "wire": "...", "diagnostics": [] }` or `{ "error": { "name": "ConfigurationFetchError", "code": "...", "message": "...", "status": 403 }, "diagnostics": [] }`. `status` is present only for HTTP failures. The adapter captures the helper's logger messages in `diagnostics` so tests can assert dropped or server-ignored selection diagnostics. Unexpected failures return HTTP 500. -Options use the Node helper's names (`distributionChannel`, `apiKey`, `clientToken`, `env`, `site`, `timeoutMs`). The test-only `abort` option supplies an already-aborted signal; `abortAfterMs` cancels an in-flight operation after the specified delay. `SYSTEM_TESTS_FFE_RULES_BASE_URL` routes the helper's custom Fetch transport to the mock backend, preserving paths, queries, headers, and redirect/cancellation options. No browser handoff is performed. +Options use the Node helper's names (`distributionChannel`, `apiKey`, `clientToken`, `env`, `site`, `flagSelection`, `timeoutMs`). The test-only `abort` option supplies an already-aborted signal; `abortAfterMs` cancels an in-flight operation after the specified delay. `SYSTEM_TESTS_FFE_RULES_BASE_URL` routes the helper's custom Fetch transport to the mock backend, preserving paths, queries, headers, and redirect/cancellation options. No browser handoff is performed. The shared `mocked_backend` fixture serves these protobuf responses. Import shared FFE constants and fixture helpers from `utils.mocked_backend.ffe`. On the server distribution endpoint, explicitly accepting `application/protobuf` selects protobuf, including in comma-separated `Accept` lists with a positive quality value. Wildcard-only requests and existing agentless provider tests continue to receive JSON. Use `mocked_backend.client.ffe_set_response` or `mocked_backend.client.ffe_set_responses` with `response_format="protobuf"` to configure these responses; the default format is JSON. Each format has an independent response timeline, and controls reject response IDs unsupported by the selected format. `mocked_backend.client.ffe_reset` resets both timelines and the shared request history. Use `mocked_backend.ffe_get_requests` to inspect paths, queries, and headers without including fixture control traffic. +The protobuf response IDs `selection_ignored_empty`, `selection_ignored_tag_load_failed`, and `selection_ignored_unknown` return the full rules fixture with a `DD-Flag-Selection-Ignored` header set to an empty string, `tag_load_failed`, or `future_reason`, respectively. Use these shared responses to test selection diagnostics without replacing the rules endpoint handler. The `valid` response omits the header. + #### Effective OpenTelemetry configuration Use `test_library.otel_effective_config()` to inspect resolved OTLP exporter timeouts. diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index c6debace8ca..febfbcf5bdb 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2706,6 +2706,7 @@ manifest: tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Split_Features: incomplete_test_app (Split-feature hook capture is currently implemented only in the Java parametric client) tests/parametric/test_ffe/test_exposure_hooks.py: incomplete_test_app (Exposure hook observations are not implemented) tests/parametric/test_ffe/test_rules_configuration.py: 7.0.0-pre # Exact development version until the first containing release is confirmed. + tests/parametric/test_ffe/test_rules_configuration.py::Test_FFE_Rules_Configuration_Selection: missing_feature (FFL-3364 requires the Node flag-selection dependency update in dd-trace-js) tests/parametric/test_ffe/test_serial_id_metadata.py: v6.18.0 tests/parametric/test_ffe/test_serial_id_metadata.py::Test_FFE_Serial_Id_Metadata_Go: irrelevant (Legacy Go metadata key) tests/parametric/test_ffe/test_span_enrichment.py: # TODO: a lower version might be supported diff --git a/tests/ffe/README.md b/tests/ffe/README.md index 8044077ca63..bf46dcbf00a 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -27,8 +27,8 @@ first, then refresh the snapshot: python3 utils/scripts/update_ffe_fixtures.py --ref ``` -The updater imports `ufc-config.json` and `evaluation-cases/*.json` only. Ordinary -test runs use those checked-in files without submodules or network access. A +The updater imports `ufc-config.json`, `evaluation-cases/*.json`, and +`flag-selection/selection-vectors.json`. Ordinary test runs use those checked-in files without submodules or network access. A weekly/manual workflow opens a signed draft PR when their contents change and leaves any existing fixture update PR untouched. Review and resolve newly exposed SDK failures before merging an update. @@ -65,6 +65,12 @@ Standalone rules-configuration helpers are covered in `tests/parametric/test_ffe These tests report to [Offline initialization and portable configuration helpers (#562)](https://feature-parity.us1.prod.dog/?feature=562). Current coverage is limited to Node's rules-based fetching and serialization; precomputed helpers and provider offline initialization are not covered here. +`Test_FFE_Rules_Configuration_Selection` covers tag/key-pattern query parameters on both distribution channels, invalid/URL-overflow selections dropped with diagnostics, quiet empty-selection omission, and retention of the full response when the backend ignores selection. It also checks diagnostics for nonempty `DD-Flag-Selection-Ignored` response headers, including unknown reasons. The mock does not implement backend filtering. These cases require [openfeature-js-client #445](https://github.com/DataDog/openfeature-js-client/pull/445) in the `dd-trace` dependency and remain gated in the Node manifest until a containing build is available. Other SDKs retain the existing file-level gate. + +The class also runs the canonical selection vectors' `sdk` expectations through real HTTP requests, grouped by distribution channel to share container startup. These cover Unicode `White_Space` trimming, UTF-8 byte ordering and length, the 31/32-value and 200/201-byte boundaries, and drop-reason precedence. Raw-query percent-decoding and server-side `selectedKeys` expectations are intentionally excluded. The fixture updater validates the SDK vector shape before replacing the snapshot, and the existing update workflow tracks changes to both evaluator and selection fixtures. + +To validate an unreleased build containing the selection change, use the [custom Node tracer instructions](../../docs/execute/binaries.md#nodejs-library), select `tests/parametric/test_ffe/test_rules_configuration.py::Test_FFE_Rules_Configuration_Selection`, and pass the same node ID to `-F`. Once released, update the class gate and the existing file-level version restriction together. + With Docker and the Python 3.12 runner installed: ```bash diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md index ec63232bc34..a957d048cf0 100644 --- a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -9,4 +9,4 @@ Do not edit these fixtures directly in system-tests. Add or update shared FFE be ffe-system-test-data first, then refresh this snapshot. The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed -draft test PR only when the server evaluator fixture contents change. +draft test PR only when the evaluator or selection fixture contents change. diff --git a/tests/parametric/test_ffe/ffe-system-test-data/flag-selection/selection-vectors.json b/tests/parametric/test_ffe/ffe-system-test-data/flag-selection/selection-vectors.json new file mode 100644 index 00000000000..2daf24b9276 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/flag-selection/selection-vectors.json @@ -0,0 +1,270 @@ +{ + "flags": [ + {"key": "checkout.new-flow", "tags": ["team:payments", "checkout"]}, + {"key": "checkout.legacy", "tags": ["team:payments"]}, + {"key": "search.ranking-v2", "tags": ["team:search"]}, + {"key": "search.autocomplete", "tags": []}, + {"key": "mobile.dark-mode", "tags": ["mobile", "ui"]}, + {"key": "a*b", "tags": ["literal"]} + ], + "cases": [ + { + "name": "no parameters selects every flag", + "query": {}, + "expected": {"outcome": "applied", "selectedKeys": ["a*b", "checkout.legacy", "checkout.new-flow", "mobile.dark-mode", "search.autocomplete", "search.ranking-v2"]}, + "sdk": {"selection": {"tags": [], "keyPatterns": []}} + }, + { + "name": "tag selects flags having any of the tags", + "query": {"selected_flag_tags": ["team:search", "mobile"]}, + "expected": {"outcome": "applied", "selectedKeys": ["mobile.dark-mode", "search.ranking-v2"]}, + "sdk": {"selection": {"tags": ["mobile", "team:search"], "keyPatterns": []}} + }, + { + "name": "tags and key patterns are combined with OR", + "query": {"selected_flag_tags": ["mobile"], "selected_flag_key_patterns": ["checkout.*"]}, + "expected": {"outcome": "applied", "selectedKeys": ["checkout.legacy", "checkout.new-flow", "mobile.dark-mode"]}, + "sdk": {"selection": {"tags": ["mobile"], "keyPatterns": ["checkout.*"]}} + }, + { + "name": "key pattern must match the whole key", + "query": {"selected_flag_key_patterns": ["checkout"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["checkout"]}} + }, + { + "name": "key pattern without a star selects exactly that key", + "query": {"selected_flag_key_patterns": ["checkout.new-flow"]}, + "expected": {"outcome": "applied", "selectedKeys": ["checkout.new-flow"]}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["checkout.new-flow"]}} + }, + { + "name": "star matches an empty run and multiple stars are allowed", + "query": {"selected_flag_key_patterns": ["*search*auto*complete*"]}, + "expected": {"outcome": "applied", "selectedKeys": ["search.autocomplete"]}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["*search*auto*complete*"]}} + }, + { + "name": "leading star", + "query": {"selected_flag_key_patterns": ["*-v2"]}, + "expected": {"outcome": "applied", "selectedKeys": ["search.ranking-v2"]}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["*-v2"]}} + }, + { + "name": "lone star matches every key", + "query": {"selected_flag_key_patterns": ["*"]}, + "expected": {"outcome": "applied", "selectedKeys": ["a*b", "checkout.legacy", "checkout.new-flow", "mobile.dark-mode", "search.autocomplete", "search.ranking-v2"]}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["*"]}} + }, + { + "name": "question mark and brackets are literal", + "query": {"selected_flag_key_patterns": ["search.ranking-v?", "[a]*", "{a,b}"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["[a]*", "search.ranking-v?", "{a,b}"]}} + }, + { + "name": "braces are literal even when an alternative names a flag", + "query": {"selected_flag_key_patterns": ["{mobile.dark-mode,missing}"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["{mobile.dark-mode,missing}"]}} + }, + { + "name": "tags are never globs", + "query": {"selected_flag_tags": ["team:*"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["team:*"], "keyPatterns": []}} + }, + { + "name": "tags match case-sensitively", + "query": {"selected_flag_tags": ["Team:Payments", "MOBILE"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["MOBILE", "Team:Payments"], "keyPatterns": []}} + }, + { + "name": "key patterns match case-sensitively", + "query": {"selected_flag_key_patterns": ["CHECKOUT.*", "Mobile.Dark-Mode"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["CHECKOUT.*", "Mobile.Dark-Mode"]}} + }, + { + "name": "values are trimmed, empties dropped and duplicates removed", + "query": {"selected_flag_tags": [" mobile ", "mobile", "", " "]}, + "expected": {"outcome": "applied", "selectedKeys": ["mobile.dark-mode"]}, + "sdk": {"selection": {"tags": ["mobile"], "keyPatterns": []}} + }, + { + "name": "lists that are empty after normalization select every flag", + "query": {"selected_flag_tags": ["", " "], "selected_flag_key_patterns": [" "]}, + "expected": {"outcome": "applied", "selectedKeys": ["a*b", "checkout.legacy", "checkout.new-flow", "mobile.dark-mode", "search.autocomplete", "search.ranking-v2"]}, + "sdk": {"selection": {"tags": [], "keyPatterns": []}} + }, + { + "name": "values are trimmed of Unicode white space", + "query": {"selected_flag_tags": ["\u00a0mobile\u3000", "\u0085ui\u2028", "\t team:search \n"]}, + "expected": {"outcome": "applied", "selectedKeys": ["mobile.dark-mode", "search.ranking-v2"]}, + "sdk": {"selection": {"tags": ["mobile", "team:search", "ui"], "keyPatterns": []}} + }, + { + "name": "characters outside Unicode White_Space are not trimmed", + "query": {"selected_flag_tags": ["\ufeffmobile", "mobile\u200b"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["mobile\u200b", "\ufeffmobile"], "keyPatterns": []}} + }, + { + "name": "values sort by UTF-8 bytes", + "query": {"selected_flag_tags": ["\ud83d\ude00", "\uff5e", "a", "Z"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["Z", "a", "\uff5e", "\ud83d\ude00"], "keyPatterns": []}} + }, + { + "name": "partial match serves only the matching flags", + "query": {"selected_flag_tags": ["mobile", "missing"], "selected_flag_key_patterns": ["nothing.*"]}, + "expected": {"outcome": "applied", "selectedKeys": ["mobile.dark-mode"]}, + "sdk": {"selection": {"tags": ["missing", "mobile"], "keyPatterns": ["nothing.*"]}} + }, + { + "name": "undecodable value is malformed", + "rawQuery": "dd_env=prod&selected_flag_tags=mobile&selected_flag_tags=%zz", + "expected": {"outcome": "ignored", "reason": "malformed_selection"} + }, + { + "name": "undecodable unrelated parameter is not a selection error", + "rawQuery": "other=%zz&selected_flag_tags=mobile", + "expected": {"outcome": "applied", "selectedKeys": ["mobile.dark-mode"]} + }, + { + "name": "repeated parameters in a raw query are all read", + "rawQuery": "selected_flag_tags=mobile&selected_flag_tags=team%3Asearch&selected_flag_key_patterns=a%2Ab", + "expected": {"outcome": "applied", "selectedKeys": ["a*b", "mobile.dark-mode", "search.ranking-v2"]} + }, + { + "name": "parameter names are percent-decoded", + "rawQuery": "selected%5Fflag%5Ftags=mobile", + "expected": {"outcome": "applied", "selectedKeys": ["mobile.dark-mode"]} + }, + { + "name": "31 values in one list are allowed", + "query": {"selected_flag_tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31"], "keyPatterns": []}} + }, + { + "name": "32 values in one list are rejected", + "query": {"selected_flag_key_patterns": ["k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18", "k19", "k20", "k21", "k22", "k23", "k24", "k25", "k26", "k27", "k28", "k29", "k30", "k31", "k32"]}, + "expected": {"outcome": "ignored", "reason": "too_many_values"}, + "sdk": {"dropped": "too_many_values"} + }, + { + "name": "31 values in each list are allowed", + "query": {"selected_flag_tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31"], "selected_flag_key_patterns": ["p01", "p02", "p03", "p04", "p05", "p06", "p07", "p08", "p09", "p10", "p11", "p12", "p13", "p14", "p15", "p16", "p17", "p18", "p19", "p20", "p21", "p22", "p23", "p24", "p25", "p26", "p27", "p28", "p29", "p30", "p31"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31"], "keyPatterns": ["p01", "p02", "p03", "p04", "p05", "p06", "p07", "p08", "p09", "p10", "p11", "p12", "p13", "p14", "p15", "p16", "p17", "p18", "p19", "p20", "p21", "p22", "p23", "p24", "p25", "p26", "p27", "p28", "p29", "p30", "p31"]}} + }, + { + "name": "32 values in one list are rejected even when the other is within the limit", + "query": {"selected_flag_tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31"], "selected_flag_key_patterns": ["p01", "p02", "p03", "p04", "p05", "p06", "p07", "p08", "p09", "p10", "p11", "p12", "p13", "p14", "p15", "p16", "p17", "p18", "p19", "p20", "p21", "p22", "p23", "p24", "p25", "p26", "p27", "p28", "p29", "p30", "p31", "p32"]}, + "expected": {"outcome": "ignored", "reason": "too_many_values"}, + "sdk": {"dropped": "too_many_values"} + }, + { + "name": "duplicates do not count toward the limit", + "query": {"selected_flag_tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31", "t01", " t02 "]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31"], "keyPatterns": []}} + }, + { + "name": "key pattern of 200 bytes is allowed", + "query": {"selected_flag_key_patterns": ["*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}} + }, + { + "name": "key pattern of 201 bytes is invalid", + "query": {"selected_flag_key_patterns": ["*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}, + "expected": {"outcome": "ignored", "reason": "invalid_key_pattern"}, + "sdk": {"dropped": "invalid_key_pattern"} + }, + { + "name": "key pattern length is measured after trimming", + "query": {"selected_flag_key_patterns": [" *xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx "]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}} + }, + { + "name": "key pattern length counts UTF-8 bytes: 100 two-byte characters are allowed", + "query": {"selected_flag_key_patterns": ["éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": [], "keyPatterns": ["éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"]}} + }, + { + "name": "key pattern length counts UTF-8 bytes: 201 bytes are invalid", + "query": {"selected_flag_key_patterns": ["xéééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"]}, + "expected": {"outcome": "ignored", "reason": "invalid_key_pattern"}, + "sdk": {"dropped": "invalid_key_pattern"} + }, + { + "name": "tag of 200 bytes is allowed", + "query": {"selected_flag_tags": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"], "keyPatterns": []}} + }, + { + "name": "tag of 201 bytes is invalid", + "query": {"selected_flag_tags": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]}, + "expected": {"outcome": "ignored", "reason": "invalid_tag"}, + "sdk": {"dropped": "invalid_tag"} + }, + { + "name": "tag length is measured after trimming", + "query": {"selected_flag_tags": [" aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa "]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"], "keyPatterns": []}} + }, + { + "name": "tag length counts UTF-8 bytes: 100 two-byte characters are allowed", + "query": {"selected_flag_tags": ["éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"]}, + "expected": {"outcome": "applied", "selectedKeys": []}, + "sdk": {"selection": {"tags": ["éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"], "keyPatterns": []}} + }, + { + "name": "tag length counts UTF-8 bytes: 201 bytes are invalid", + "query": {"selected_flag_tags": ["xéééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"]}, + "expected": {"outcome": "ignored", "reason": "invalid_tag"}, + "sdk": {"dropped": "invalid_tag"} + }, + { + "name": "a long tag invalidates the selection even when other tags match", + "query": {"selected_flag_tags": ["mobile", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]}, + "expected": {"outcome": "ignored", "reason": "invalid_tag"}, + "sdk": {"dropped": "invalid_tag"} + }, + { + "name": "too_many_values takes precedence over invalid_key_pattern", + "query": {"selected_flag_tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31", "t32"], "selected_flag_key_patterns": ["*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}, + "expected": {"outcome": "ignored", "reason": "too_many_values"}, + "sdk": {"dropped": "too_many_values"} + }, + { + "name": "malformed_selection takes precedence over too_many_values", + "rawQuery": "selected_flag_tags=t01&selected_flag_tags=t02&selected_flag_tags=t03&selected_flag_tags=t04&selected_flag_tags=t05&selected_flag_tags=t06&selected_flag_tags=t07&selected_flag_tags=t08&selected_flag_tags=t09&selected_flag_tags=t10&selected_flag_tags=t11&selected_flag_tags=t12&selected_flag_tags=t13&selected_flag_tags=t14&selected_flag_tags=t15&selected_flag_tags=t16&selected_flag_tags=t17&selected_flag_tags=t18&selected_flag_tags=t19&selected_flag_tags=t20&selected_flag_tags=t21&selected_flag_tags=t22&selected_flag_tags=t23&selected_flag_tags=t24&selected_flag_tags=t25&selected_flag_tags=t26&selected_flag_tags=t27&selected_flag_tags=t28&selected_flag_tags=t29&selected_flag_tags=t30&selected_flag_tags=t31&selected_flag_tags=t32&selected_flag_key_patterns=%zz", + "expected": {"outcome": "ignored", "reason": "malformed_selection"} + }, + { + "name": "too_many_values takes precedence over invalid_tag", + "query": {"selected_flag_tags": ["t01", "t02", "t03", "t04", "t05", "t06", "t07", "t08", "t09", "t10", "t11", "t12", "t13", "t14", "t15", "t16", "t17", "t18", "t19", "t20", "t21", "t22", "t23", "t24", "t25", "t26", "t27", "t28", "t29", "t30", "t31", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]}, + "expected": {"outcome": "ignored", "reason": "too_many_values"}, + "sdk": {"dropped": "too_many_values"} + }, + { + "name": "invalid_key_pattern takes precedence over invalid_tag", + "query": {"selected_flag_tags": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"], "selected_flag_key_patterns": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]}, + "expected": {"outcome": "ignored", "reason": "invalid_key_pattern"}, + "sdk": {"dropped": "invalid_key_pattern"} + }, + { + "name": "malformed_selection takes precedence over invalid_tag", + "rawQuery": "selected_flag_tags=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&selected_flag_tags=%zz", + "expected": {"outcome": "ignored", "reason": "malformed_selection"} + } + ] +} diff --git a/tests/parametric/test_ffe/test_rules_configuration.py b/tests/parametric/test_ffe/test_rules_configuration.py index 52b4057da7f..8a550f77344 100644 --- a/tests/parametric/test_ffe/test_rules_configuration.py +++ b/tests/parametric/test_ffe/test_rules_configuration.py @@ -2,6 +2,7 @@ import base64 import json +from pathlib import Path from typing import Any from urllib.parse import parse_qs, urlsplit @@ -14,6 +15,7 @@ CLIENT_TOKEN = "system-tests-client-token" API_KEY = "system-tests-api-key" CLIENT_OPTIONS = {"distributionChannel": "client", "clientToken": CLIENT_TOKEN, "env": "test env"} +SELECTION_VECTORS_PATH = Path(__file__).parent / "ffe-system-test-data" / "flag-selection" / "selection-vectors.json" @pytest.fixture @@ -161,3 +163,242 @@ def test_reject_invalid_options(self, test_library: APMLibrary, mocked_backend: assert CLIENT_TOKEN not in json.dumps(result), case assert API_KEY not in json.dumps(result), case assert mocked_backend.ffe_get_requests() == [], case + + +@scenarios.parametric +@features.feature_flags_portable_configuration +@slow +class Test_FFE_Rules_Configuration_Selection: + @pytest.mark.parametrize("channel", ["client", "server"]) + def test_canonical_selection_vectors( + self, test_library: APMLibrary, mocked_backend: MockedBackendServer, channel: str + ) -> None: + vectors = json.loads(SELECTION_VECTORS_PATH.read_text(encoding="utf-8")) + # Raw-query cases exercise backend percent-decoding, not an SDK option. + sdk_cases = [vector for vector in vectors["cases"] if "sdk" in vector] + assert sdk_cases, "No canonical SDK selection vectors found" + credential = {"clientToken": CLIENT_TOKEN} if channel == "client" else {"apiKey": API_KEY} + options = {"distributionChannel": channel, "env": "test env", **credential} + with test_library: + baseline = test_library.ffe_fetch_rules_configuration(options) + assert "error" not in baseline, (channel, baseline) + assert baseline["diagnostics"] == [], (channel, baseline) + original = json.loads(baseline["wire"]) + for vector in sdk_cases: + case = (channel, vector["name"]) + selection = { + field: vector["query"][parameter] + for field, parameter in ( + ("tags", "selected_flag_tags"), + ("keyPatterns", "selected_flag_key_patterns"), + ) + if parameter in vector["query"] + } + request_count = len(mocked_backend.ffe_get_requests()) + result = test_library.ffe_fetch_rules_configuration({**options, "flagSelection": selection}) + assert "error" not in result, (case, result) + requests = mocked_backend.ffe_get_requests()[request_count:] + assert len(requests) == 1, (case, requests) + url = urlsplit(requests[0]["path"]) + assert url.path == f"{RULES_CONFIG_PATH}/{channel}", case + expected_query = {"dd_env": ["test env"]} + sdk = vector["sdk"] + if "dropped" in sdk: + assert len(result["diagnostics"]) == 1, (case, result) + assert result["diagnostics"][0].startswith(f"flagSelection was not sent ({sdk['dropped']}: "), ( + case, + result, + ) + assert result["diagnostics"][0].endswith("). Loading the full flag configuration."), (case, result) + else: + for field, parameter in ( + ("tags", "selected_flag_tags"), + ("keyPatterns", "selected_flag_key_patterns"), + ): + if sdk["selection"][field]: + expected_query[parameter] = sdk["selection"][field] + assert result["diagnostics"] == [], (case, result) + assert parse_qs(url.query, keep_blank_values=True) == expected_query, (case, url.query) + + # The mock deliberately returns its full configuration; do not test + # the vectors' server-side selectedKeys using an SDK-side filter. + wire = json.loads(result["wire"]) + assert wire["version"] == original["version"], case + assert wire["rules"]["response"] == original["rules"]["response"], case + assert wire["rules"]["etag"] == original["rules"]["etag"], case + assert CLIENT_TOKEN not in json.dumps(result), case + assert API_KEY not in json.dumps(result), case + + def test_selection_query_and_unfiltered_response( + self, test_library: APMLibrary, mocked_backend: MockedBackendServer + ) -> None: + with test_library: + for channel, credential in (("client", {"clientToken": CLIENT_TOKEN}), ("server", {"apiKey": API_KEY})): + options = {"distributionChannel": channel, "env": "test env", **credential} + baseline = test_library.ffe_fetch_rules_configuration(options) + request_count = len(mocked_backend.ffe_get_requests()) + result = test_library.ffe_fetch_rules_configuration( + { + **options, + "flagSelection": { + "tags": [" team:checkout ", "release:summer sale", "team:checkout"], + "keyPatterns": [" checkout-banner ", "checkout-*", "checkout-banner"], + }, + } + ) + + assert "error" not in result, (channel, result) + requests = mocked_backend.ffe_get_requests()[request_count:] + assert len(requests) == 1, (channel, requests) + url = urlsplit(requests[0]["path"]) + assert url.path == f"{RULES_CONFIG_PATH}/{channel}", channel + assert parse_qs(url.query, keep_blank_values=True) == { + "dd_env": ["test env"], + "selected_flag_tags": ["release:summer sale", "team:checkout"], + "selected_flag_key_patterns": ["checkout-*", "checkout-banner"], + }, (channel, url.query) + assert "release%3Asummer%20sale" in url.query, channel + assert "checkout-%2A" in url.query, channel + headers = requests[0]["headers"] + assert headers.get("dd-client-token") == (CLIENT_TOKEN if channel == "client" else None), channel + assert headers.get("dd-api-key") == (API_KEY if channel == "server" else None), channel + + # The mock ignores selection. Retain its full response even though test-flag + # does not match either requested key pattern. + wire = json.loads(result["wire"]) + original = json.loads(baseline["wire"]) + assert wire["version"] == original["version"], channel + assert wire["rules"]["response"] == original["rules"]["response"], channel + assert wire["rules"]["etag"] == original["rules"]["etag"], channel + assert result["diagnostics"] == [], (channel, result) + + def test_server_ignored_selection_diagnostics( + self, test_library: APMLibrary, mocked_backend: MockedBackendServer + ) -> None: + with test_library: + for channel, credential in (("client", {"clientToken": CLIENT_TOKEN}), ("server", {"apiKey": API_KEY})): + mocked_backend.client.ffe_set_response("valid", response_format="protobuf") + options = {"distributionChannel": channel, "env": "test env", **credential} + baseline = test_library.ffe_fetch_rules_configuration(options) + original = json.loads(baseline["wire"]) + assert baseline["diagnostics"] == [], (channel, baseline) + for response, ignored_reason in ( + ("selection_ignored_empty", ""), + ("selection_ignored_tag_load_failed", "tag_load_failed"), + ("selection_ignored_unknown", "future_reason"), + ): + mocked_backend.client.ffe_set_response(response, response_format="protobuf") + request_count = len(mocked_backend.ffe_get_requests()) + result = test_library.ffe_fetch_rules_configuration( + {**options, "flagSelection": {"keyPatterns": ["non-matching-*"]}} + ) + assert "error" not in result, (channel, ignored_reason, result) + requests = mocked_backend.ffe_get_requests()[request_count:] + assert len(requests) == 1, (channel, ignored_reason, requests) + url = urlsplit(requests[0]["path"]) + assert url.path == f"{RULES_CONFIG_PATH}/{channel}", (channel, ignored_reason) + assert parse_qs(url.query, keep_blank_values=True) == { + "dd_env": ["test env"], + "selected_flag_key_patterns": ["non-matching-*"], + }, (channel, ignored_reason, requests[0]) + wire = json.loads(result["wire"]) + assert wire["version"] == original["version"], channel + assert wire["rules"]["response"] == original["rules"]["response"], channel + assert wire["rules"]["etag"] == original["rules"]["etag"], channel + assert result["diagnostics"] == ( + [ + f"flagSelection was ignored by the server ({ignored_reason}). Loaded the full flag configuration." + ] + if ignored_reason + else [] + ), (channel, ignored_reason, result) + + def test_empty_and_url_overflow_selections_are_not_sent( + self, test_library: APMLibrary, mocked_backend: MockedBackendServer + ) -> None: + cases = { + "empty": {"tags": [], "keyPatterns": []}, + # Each list and value is valid; the combined encoded URL exceeds 8192 bytes. + "url-overflow": { + "tags": [f"team:{index}:" + "a" * 180 for index in range(31)], + "keyPatterns": [f"flag-{index}-" + "b" * 180 + "*" for index in range(31)], + }, + } + with test_library: + for channel, credential in (("client", {"clientToken": CLIENT_TOKEN}), ("server", {"apiKey": API_KEY})): + for case, selection in cases.items(): + request_count = len(mocked_backend.ffe_get_requests()) + result = test_library.ffe_fetch_rules_configuration( + {"distributionChannel": channel, "env": "test env", **credential, "flagSelection": selection} + ) + assert "error" not in result, (channel, case, result) + assert json.loads(result["wire"])["rules"]["response"], (channel, case) + requests = mocked_backend.ffe_get_requests()[request_count:] + assert len(requests) == 1, (channel, case, requests) + url = urlsplit(requests[0]["path"]) + assert url.path == f"{RULES_CONFIG_PATH}/{channel}", (channel, case) + assert parse_qs(url.query, keep_blank_values=True) == {"dd_env": ["test env"]}, ( + channel, + case, + url.query, + ) + assert result["diagnostics"] == ( + [ + "flagSelection was not sent (too_many_values: the request URL is at most 8192 bytes). Loading the full flag configuration." + ] + if case == "url-overflow" + else [] + ), (channel, case, result) + + def test_invalid_selection_is_not_sent(self, test_library: APMLibrary, mocked_backend: MockedBackendServer) -> None: + cases = [ + ("malformed-tags", {"tags": "private-selection-value"}, "malformed_selection"), + ( + "too-many-tags", + {"tags": [f"private-selection-value-tag:{index}" for index in range(32)]}, + "too_many_values", + ), + ( + "too-many-patterns", + {"keyPatterns": [f"private-selection-value-flag-{index}" for index in range(32)]}, + "too_many_values", + ), + ("oversized-tag", {"tags": ["private-selection-value".ljust(201, "x")]}, "invalid_tag"), + ("oversized-pattern", {"keyPatterns": ["private-selection-value".ljust(201, "x")]}, "invalid_key_pattern"), + ] + with test_library: + for channel, credential in (("client", {"clientToken": CLIENT_TOKEN}), ("server", {"apiKey": API_KEY})): + options = {"distributionChannel": channel, "env": "test env", **credential} + baseline = test_library.ffe_fetch_rules_configuration(options) + original = json.loads(baseline["wire"]) + for case, selection, reason in cases: + request_count = len(mocked_backend.ffe_get_requests()) + result = test_library.ffe_fetch_rules_configuration({**options, "flagSelection": selection}) + assert "error" not in result, (channel, case, result) + wire = json.loads(result["wire"]) + assert wire["version"] == original["version"], (channel, case) + assert wire["rules"]["response"] == original["rules"]["response"], (channel, case) + assert wire["rules"]["etag"] == original["rules"]["etag"], (channel, case) + assert len(result["diagnostics"]) == 1, (channel, case, result) + assert result["diagnostics"][0].startswith(f"flagSelection was not sent ({reason}: "), ( + channel, + case, + result, + ) + assert result["diagnostics"][0].endswith("). Loading the full flag configuration."), ( + channel, + case, + result, + ) + assert "private-selection-value" not in json.dumps(result), (channel, case) + assert CLIENT_TOKEN not in json.dumps(result), (channel, case) + assert API_KEY not in json.dumps(result), (channel, case) + requests = mocked_backend.ffe_get_requests()[request_count:] + assert len(requests) == 1, (channel, case, requests) + url = urlsplit(requests[0]["path"]) + assert url.path == f"{RULES_CONFIG_PATH}/{channel}", (channel, case) + assert parse_qs(url.query, keep_blank_values=True) == {"dd_env": ["test env"]}, ( + channel, + case, + url.query, + ) diff --git a/tests/test_the_test/test_mock_ffe_rules_backend.py b/tests/test_the_test/test_mock_ffe_rules_backend.py index ced7d8d7514..ea11ca21010 100644 --- a/tests/test_the_test/test_mock_ffe_rules_backend.py +++ b/tests/test_the_test/test_mock_ffe_rules_backend.py @@ -99,6 +99,9 @@ def test_server_negotiates_format_with_independent_response_timelines(self) -> N ("json", "delayed_body"), ("json", "disconnect"), ("json", "truncated_body"), + ("json", "selection_ignored_empty"), + ("json", "selection_ignored_tag_load_failed"), + ("json", "selection_ignored_unknown"), ("protobuf", "unauthorized"), ("protobuf", "not_modified"), ("protobuf", "delayed_valid"), @@ -172,6 +175,42 @@ def test_protobuf_and_request_capture(self) -> None: finally: server.close() + @pytest.mark.parametrize("channel", ["client", "server"]) + def test_selection_ignored_responses(self, channel: str) -> None: + server = MockedBackendServer(port=0) + try: + url = f"{server.base_url}{RULES_CONFIG_PATH}/{channel}" + cases = [ + ("valid", None), + ("selection_ignored_empty", ""), + ("selection_ignored_tag_load_failed", "tag_load_failed"), + ("selection_ignored_unknown", "future_reason"), + ("valid", None), + ] + server.client.ffe_set_responses([mode for mode, _ in cases], response_format="protobuf") + for _, reason in cases: + response = requests.get( + url, + params={"dd_env": "test", "selected_flag_key_patterns": "non-matching-*"}, + headers={"Accept": "application/protobuf"}, + timeout=5, + ) + assert response.status_code == 200 + assert response.headers.get("DD-Flag-Selection-Ignored") == reason + assert response.headers["Content-Type"] == "application/protobuf" + assert response.headers["ETag"] == WIRE_FIXTURE["rules"]["etag"] + assert response.content == base64.b64decode(WIRE_FIXTURE["rules"]["response"]) + assert len(server.ffe_get_requests()) == len(cases) + assert server.client.ffe_get_status()["status_codes"] == [200] * len(cases) + + server.client.ffe_set_response("selection_ignored_tag_load_failed", response_format="protobuf") + server.client.ffe_reset() + response = requests.get(url, headers={"Accept": "application/protobuf"}, timeout=5) + assert "DD-Flag-Selection-Ignored" not in response.headers + assert len(server.ffe_get_requests()) == 1 + finally: + server.close() + @pytest.mark.parametrize( ("mode", "status", "content_type", "body"), [ diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py index 8f53acd82ee..bca1db6bf6f 100644 --- a/tests/test_the_test/test_update_ffe_fixtures.py +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -1,7 +1,9 @@ # ruff: noqa: SLF001 from pathlib import Path +import json import shutil +from typing import Any from utils import pytest @@ -9,6 +11,17 @@ from utils.scripts import update_ffe_fixtures +def _write_selection_vectors(root: Path, cases: list[dict[str, Any]] | None = None) -> None: + path = root / update_ffe_fixtures.SELECTION_VECTORS_PATH + path.parent.mkdir(parents=True, exist_ok=True) + if cases is None: + cases = [ + {"name": "empty selection", "query": {}, "sdk": {"selection": {"tags": [], "keyPatterns": []}}}, + {"name": "raw query", "rawQuery": "selected_flag_tags=%zz"}, + ] + path.write_text(json.dumps({"cases": cases}), encoding="utf-8") + + @scenarios.test_the_test @pytest.mark.parametrize("fixture_ref", ["main", "feature/fixture-update", "v1.2.3", "ea8b5cc5"]) def test_validate_fixture_ref_accepts_git_refs(fixture_ref: str) -> None: @@ -23,7 +36,7 @@ def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: @scenarios.test_the_test -def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path) -> None: +def test_copy_fixture_snapshot_imports_only_consumed_fixture_data(tmp_path: Path) -> None: source = tmp_path / "source" snapshot = tmp_path / "snapshot" (source / "evaluation-cases").mkdir(parents=True) @@ -33,14 +46,28 @@ def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path (source / "README.md").write_text("upstream documentation", encoding="utf-8") (source / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") (source / "precomputed-assignments").mkdir() + suite = source / "evaluation-suites" / "feature" + (suite / "cases").mkdir(parents=True) + (suite / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + (suite / "cases" / "case.json").write_text('[{"flag": "flag"}]', encoding="utf-8") + _write_selection_vectors(source) + (source / "flag-selection" / "README.md").write_text("upstream documentation", encoding="utf-8") + (source / "flag-selection" / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) assert (snapshot / "ufc-config.json").is_file() assert (snapshot / "evaluation-cases" / "case.json").is_file() + for path in (Path("ufc-config.json"), Path("cases/case.json")): + assert (snapshot / "evaluation-suites" / "feature" / path).read_bytes() == (suite / path).read_bytes() assert not (snapshot / "README.md").exists() assert not (snapshot / "AGENTS.md").exists() assert not (snapshot / "precomputed-assignments").exists() + assert (snapshot / update_ffe_fixtures.SELECTION_VECTORS_PATH).read_bytes() == ( + source / update_ffe_fixtures.SELECTION_VECTORS_PATH + ).read_bytes() + assert not (snapshot / "flag-selection" / "README.md").exists() + assert not (snapshot / "flag-selection" / "AGENTS.md").exists() @scenarios.test_the_test @@ -59,6 +86,26 @@ def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) +@scenarios.test_the_test +@pytest.mark.parametrize("link_directory", [False, True], ids=["file", "directory"]) +def test_copy_fixture_snapshot_rejects_selection_symbolic_links(tmp_path: Path, *, link_directory: bool) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + target = tmp_path / "target" + _write_selection_vectors(target) + if link_directory: + (source / "flag-selection").symlink_to(target / "flag-selection", target_is_directory=True) + else: + (source / "flag-selection").mkdir() + (source / update_ffe_fixtures.SELECTION_VECTORS_PATH).symlink_to( + target / update_ffe_fixtures.SELECTION_VECTORS_PATH + ) + with pytest.raises(ValueError, match="Refusing to copy symbolic link"): + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + @scenarios.test_the_test def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: cases_directory = tmp_path / "evaluation-cases" @@ -70,6 +117,52 @@ def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 +@scenarios.test_the_test +def test_validate_selection_vectors_counts_only_sdk_cases(tmp_path: Path) -> None: + _write_selection_vectors(tmp_path) + assert update_ffe_fixtures._validate_selection_vectors(tmp_path) == 1 + + +@scenarios.test_the_test +@pytest.mark.parametrize( + ("cases", "message"), + [ + ([], "non-empty cases array"), + ([{"name": "raw", "rawQuery": ""}], "No FFE selection SDK cases"), + ([{"query": {}}], "non-empty name"), + ([{"name": "no query"}], "exactly one query form"), + ([{"name": "both", "query": {}, "rawQuery": ""}], "exactly one query form"), + ([{"name": "missing SDK expectation", "query": {}}], "one SDK expectation"), + ([{"name": "raw with SDK expectation", "rawQuery": "", "sdk": {}}], "must not have an SDK expectation"), + ([{"name": "invalid query", "query": {"selected_flag_tags": "web"}}], "Invalid FFE selection vector query"), + ([{"name": "invalid item", "query": {"selected_flag_tags": [1]}}], "Invalid FFE selection vector query"), + ([{"name": "unknown reason", "query": {}, "sdk": {"dropped": "unknown"}}], "Unknown FFE selection drop reason"), + ( + [{"name": "both expectations", "query": {}, "sdk": {"dropped": "invalid_tag", "selection": {}}}], + "one SDK expectation", + ), + ( + [{"name": "missing lists", "query": {}, "sdk": {"selection": {}}}], + "Invalid canonical FFE selection expectation", + ), + ( + [{"name": "empty value", "query": {}, "sdk": {"selection": {"tags": [""], "keyPatterns": []}}}], + "Invalid canonical FFE selection expectation", + ), + ( + [{"name": "duplicate", "query": {}, "sdk": {"dropped": "invalid_tag"}}] * 2, + "Duplicate FFE selection vector name", + ), + ], +) +def test_validate_selection_vectors_rejects_invalid_cases( + tmp_path: Path, cases: list[dict[str, Any]], message: str +) -> None: + _write_selection_vectors(tmp_path, cases) + with pytest.raises(ValueError, match=message): + update_ffe_fixtures._validate_selection_vectors(tmp_path) + + @scenarios.test_the_test def test_have_same_contents_ignores_generated_source_metadata(tmp_path: Path) -> None: snapshot = tmp_path / "snapshot" @@ -87,14 +180,18 @@ def test_have_same_contents_ignores_generated_source_metadata(tmp_path: Path) -> @scenarios.test_the_test +@pytest.mark.parametrize("invalid_selection", [False, True], ids=["evaluation", "selection"]) def test_update_fixtures_preserves_snapshot_on_validation_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, invalid_selection: bool ) -> None: source = tmp_path / "upstream" destination = tmp_path / "destination" (source / "evaluation-cases").mkdir(parents=True) (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") - (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + (source / "evaluation-cases" / "case.json").write_text( + '[{"flag": "flag"}]' if invalid_selection else "[]", encoding="utf-8" + ) + _write_selection_vectors(source, [] if invalid_selection else None) destination.mkdir() (destination / "SOURCE.md").write_text("original snapshot", encoding="utf-8") @@ -105,7 +202,9 @@ def fake_git(working_directory: Path, *arguments: str) -> str: monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) - with pytest.raises(ValueError, match="at least one evaluation case"): + with pytest.raises( + ValueError, match="non-empty cases array" if invalid_selection else "at least one evaluation case" + ): update_ffe_fixtures.update_fixtures("main") assert (destination / "SOURCE.md").read_text() == "original snapshot" @@ -121,6 +220,7 @@ def test_update_fixtures_records_source_and_removes_stale_files( (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") case_file = source / "evaluation-cases" / "case.json" case_file.write_text('[{"flag": "flag"}]', encoding="utf-8") + _write_selection_vectors(source) destination.mkdir() (destination / "stale.json").write_text("[]", encoding="utf-8") @@ -137,9 +237,16 @@ def fake_git(working_directory: Path, *arguments: str) -> str: assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() metadata = (destination / "SOURCE.md").read_bytes() assert ("a" * 40).encode() in metadata - assert "fixture_count=1\nchanged=true\n" in output.read_text() + assert "fixture_count=1\nselection_fixture_count=1\nchanged=true\n" in output.read_text() (source / "README.md").write_text("documentation-only upstream change", encoding="utf-8") assert not update_ffe_fixtures.update_fixtures("main") assert (destination / "SOURCE.md").read_bytes() == metadata assert output.read_text().endswith("changed=false\n") + + _write_selection_vectors(source, [{"name": "new SDK case", "query": {}, "sdk": {"dropped": "invalid_tag"}}]) + assert update_ffe_fixtures.update_fixtures("main") + assert (destination / update_ffe_fixtures.SELECTION_VECTORS_PATH).read_bytes() == ( + source / update_ffe_fixtures.SELECTION_VECTORS_PATH + ).read_bytes() + assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() diff --git a/utils/build/docker/nodejs/parametric/ffe-rules-configuration.js b/utils/build/docker/nodejs/parametric/ffe-rules-configuration.js index d17fbd38db6..4d4a5359b4d 100644 --- a/utils/build/docker/nodejs/parametric/ffe-rules-configuration.js +++ b/utils/build/docker/nodejs/parametric/ffe-rules-configuration.js @@ -9,6 +9,7 @@ const { async function main () { const { abort = false, abortAfterMs, ...options } = JSON.parse(process.argv[2]) const controller = new AbortController() + const diagnostics = [] if (abort) controller.abort() const abortTimer = abortAfterMs === undefined ? undefined @@ -17,6 +18,7 @@ async function main () { try { const configuration = await fetchRulesConfiguration({ ...options, + logger: { error: message => diagnostics.push(message) }, signal: controller.signal, // Exercise real HTTP while keeping generated paths, headers, and fetch options intact. fetch: (url, init) => { @@ -24,11 +26,12 @@ async function main () { return fetch(`${process.env.SYSTEM_TESTS_FFE_RULES_BASE_URL}${pathname}${search}`, init) } }) - process.stdout.write(JSON.stringify({ wire: configurationToString(configuration) })) + process.stdout.write(JSON.stringify({ wire: configurationToString(configuration), diagnostics })) } catch (error) { if (!(error instanceof ConfigurationFetchError)) throw error process.stdout.write(JSON.stringify({ - error: { name: error.name, code: error.code, message: error.message, status: error.status } + error: { name: error.name, code: error.code, message: error.message, status: error.status }, + diagnostics })) } finally { clearTimeout(abortTimer) diff --git a/utils/mocked_backend/_backend_ffe.py b/utils/mocked_backend/_backend_ffe.py index f289b45c39e..8f7a6b2a8ae 100644 --- a/utils/mocked_backend/_backend_ffe.py +++ b/utils/mocked_backend/_backend_ffe.py @@ -33,6 +33,11 @@ _Base = object ResponseFormat = Literal["json", "protobuf"] +SELECTION_IGNORED_RESPONSES = { + "selection_ignored_empty": "", + "selection_ignored_tag_load_failed": "tag_load_failed", + "selection_ignored_unknown": "future_reason", +} VALID_RESPONSE_IDS: dict[ResponseFormat, frozenset[str]] = { "json": frozenset( { @@ -58,6 +63,7 @@ "delayed_body", "disconnect", "truncated_body", + *SELECTION_IGNORED_RESPONSES, } ), } @@ -368,7 +374,9 @@ def _ffe_handle_rules_config( wire_fixture = load_rules_wire_fixture() headers = {"Content-Type": "application/protobuf", "ETag": wire_fixture["rules"]["etag"]} body = base64.b64decode(wire_fixture["rules"]["response"]) - if response == "delayed": + if response in SELECTION_IGNORED_RESPONSES: + headers["DD-Flag-Selection-Ignored"] = SELECTION_IGNORED_RESPONSES[response] + elif response == "delayed": time.sleep(4) elif response == "delayed_body": # Each phase fits within the deadline, but their combined duration does not. diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py index 9b9fbf2289e..4a41aaf2cdf 100644 --- a/utils/scripts/update_ffe_fixtures.py +++ b/utils/scripts/update_ffe_fixtures.py @@ -14,6 +14,8 @@ FIXTURE_DESTINATION = REPOSITORY_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" FIXTURE_REPOSITORY = "https://github.com/DataDog/ffe-system-test-data.git" FIXTURE_REF_PATTERN = re.compile(r"[A-Za-z0-9._/-]+") +SELECTION_VECTORS_PATH = Path("flag-selection/selection-vectors.json") +SELECTION_DROP_REASONS = frozenset({"malformed_selection", "too_many_values", "invalid_key_pattern", "invalid_tag"}) def _validate_fixture_ref(fixture_ref: str) -> None: @@ -64,10 +66,13 @@ def _copy_entry(source: Path, destination: Path) -> None: def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: - # Import only rules-based evaluator corpora. Precomputed assignments use a - # different format and are consumed by a different test harness. + # Import rules-based evaluator corpora and selection vectors, not precomputed assignments. for name in ("ufc-config.json", "evaluation-cases"): _copy_entry(source / name, snapshot / name) + if (source / SELECTION_VECTORS_PATH.parent).is_symlink(): + msg = "Refusing to copy symbolic link from FFE fixture repository: flag-selection" + raise ValueError(msg) + _copy_entry(source / SELECTION_VECTORS_PATH, snapshot / SELECTION_VECTORS_PATH) feature_suites = source / "evaluation-suites" if feature_suites.exists() or feature_suites.is_symlink(): @@ -136,6 +141,73 @@ def _validate_fixture_snapshot(snapshot: Path) -> int: return fixture_count +def _validate_selection_vectors(snapshot: Path) -> int: + with (snapshot / SELECTION_VECTORS_PATH).open(encoding="utf-8") as fixture_file: + vectors = json.load(fixture_file) + cases = vectors.get("cases") if isinstance(vectors, dict) else None + if not isinstance(cases, list) or not cases: + msg = "FFE selection vectors must contain a non-empty cases array" + raise ValueError(msg) + + names: set[str] = set() + sdk_count = 0 + for case in cases: + if not isinstance(case, dict) or not isinstance(case.get("name"), str) or not case["name"]: + msg = "FFE selection vector must have a non-empty name" + raise ValueError(msg) + name = case["name"] + if name in names: + msg = f"Duplicate FFE selection vector name: {name}" + raise ValueError(msg) + names.add(name) + if ("query" in case) == ("rawQuery" in case): + msg = f"FFE selection vector must have exactly one query form: {name}" + raise ValueError(msg) + if "rawQuery" in case: + if not isinstance(case["rawQuery"], str) or "sdk" in case: + msg = f"Raw-query FFE selection vector must not have an SDK expectation: {name}" + raise ValueError(msg) + continue + + query = case["query"] + if ( + not isinstance(query, dict) + or set(query) - {"selected_flag_tags", "selected_flag_key_patterns"} + or any( + not isinstance(values, list) or any(not isinstance(value, str) for value in values) + for values in query.values() + ) + ): + msg = f"Invalid FFE selection vector query: {name}" + raise ValueError(msg) + sdk = case.get("sdk") + if not isinstance(sdk, dict) or set(sdk) not in ({"selection"}, {"dropped"}): + msg = f"FFE selection vector must have one SDK expectation: {name}" + raise ValueError(msg) + if "dropped" in sdk: + if not isinstance(sdk["dropped"], str) or sdk["dropped"] not in SELECTION_DROP_REASONS: + msg = f"Unknown FFE selection drop reason: {name}" + raise ValueError(msg) + else: + selection = sdk["selection"] + if ( + not isinstance(selection, dict) + or set(selection) != {"tags", "keyPatterns"} + or any( + not isinstance(values, list) or any(not isinstance(value, str) or not value for value in values) + for values in selection.values() + ) + ): + msg = f"Invalid canonical FFE selection expectation: {name}" + raise ValueError(msg) + sdk_count += 1 + + if sdk_count == 0: + msg = "No FFE selection SDK cases found" + raise ValueError(msg) + return sdk_count + + def _relative_files(directory: Path, excluded_files: frozenset[str] = frozenset()) -> list[Path]: return sorted( path.relative_to(directory) @@ -172,18 +244,19 @@ def _write_source_metadata(snapshot: Path, source_commit: str) -> None: ffe-system-test-data first, then refresh this snapshot. The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed -draft test PR only when the server evaluator fixture contents change. +draft test PR only when the evaluator or selection fixture contents change. """, encoding="utf-8", ) -def _set_github_outputs(source_commit: str, fixture_count: int, *, changed: bool) -> None: +def _set_github_outputs(source_commit: str, fixture_count: int, selection_fixture_count: int, *, changed: bool) -> None: github_output = os.environ.get("GITHUB_OUTPUT") if github_output: with Path(github_output).open("a", encoding="utf-8") as output_file: output_file.write(f"source_commit={source_commit}\n") output_file.write(f"fixture_count={fixture_count}\n") + output_file.write(f"selection_fixture_count={selection_fixture_count}\n") output_file.write(f"changed={str(changed).lower()}\n") @@ -206,6 +279,7 @@ def update_fixtures(fixture_ref: str) -> bool: _copy_fixture_snapshot(source, snapshot) fixture_count = _validate_fixture_snapshot(snapshot) + selection_fixture_count = _validate_selection_vectors(snapshot) changed = not _have_same_contents(snapshot, FIXTURE_DESTINATION) if changed: @@ -214,9 +288,10 @@ def update_fixtures(fixture_ref: str) -> bool: shutil.rmtree(FIXTURE_DESTINATION) shutil.copytree(snapshot, FIXTURE_DESTINATION) - _set_github_outputs(source_commit, fixture_count, changed=changed) + _set_github_outputs(source_commit, fixture_count, selection_fixture_count, changed=changed) print(f"Checked FFE fixtures from DataDog/ffe-system-test-data@{source_commit}") print(f"Loaded {fixture_count} canonical fixture cases") + print(f"Loaded {selection_fixture_count} canonical selection SDK cases") print(f"Fixture snapshot changed: {str(changed).lower()}") return changed