diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3af4038..336e7a23 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,8 @@ When a new feature's UFC syntax itself is not supported by every SDK, create a self-contained `evaluation-suites//` directory containing its own `ufc-config.json` and `cases/`. Do not add that syntax to the legacy root config: a suite the consumer has not enabled must not have its configuration parsed. +A suite may also hold `flag-tags.json` when its server double needs flag tags, +as `flag-selection` does; see [flag selection suite](README.md#flag-selection-suite). For selected split/allocation metadata, add optional `result.serialId` (integer or `null`) and `result.doLog` (boolean) expectations. A serial ID of `0` is valid; diff --git a/README.md b/README.md index dcfdbc6c..3a6df2ca 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,13 @@ ffe-system-test-data/ ├── evaluation-cases/ │ └── test-*.json # Evaluation test case files ├── evaluation-suites/ # Optional self-contained, explicitly activated feature suites -│ └── dependent-flags/ -│ ├── ufc-config.json # UFC containing only dependent-flag fixtures -│ └── cases/ # Evaluation cases for this feature +│ ├── dependent-flags/ +│ │ ├── ufc-config.json # UFC containing only dependent-flag fixtures +│ │ └── cases/ # Evaluation cases for this feature +│ └── flag-selection/ +│ ├── ufc-config.json # Flags a server double narrows to the SDK's flag selection +│ ├── flag-tags.json # Tags of those flags, which UFC does not carry +│ └── cases/ # One flag selection per case file ├── precomputed-assignments/ │ ├── README.md # Precomputed assignment fixture format │ └── cases/ # Precomputed assignment fixtures, discovered recursively @@ -230,6 +234,9 @@ Each evaluation case uses a universal schema with the following fields: | `result.doLog` | boolean | Optional expected selected allocation's logging decision | | `result.features` | object | Optional exact feature maps for `HOOK`, `EXPOSURE`, and `EVALUATION`; empty maps assert no features, omission leaves routing unspecified | | `capabilities` | object | Optional expected results when a named capability is enabled; uses the same result format | +| `providerOptions.flagSelection` | object | Optional raw flag selection, `{"tags": [...], "keyPatterns": [...]}`, passed untouched to the SDK's flag selection provider option. See [flag selection suite](#flag-selection-suite) | +| `expectations.configurationRequest` | object | Optional selection the SDK sends on its configuration request: canonical `selectedFlagTags` and `selectedFlagKeyPatterns` lists, or `{"flagSelection": null}` when it sends none | +| `configurationResponse.flagSelectionIgnored` | string | Optional reason the server double gives for ignoring the valid selection it receives | | `_skip` | array | Optional platforms that exclude this case | | `_include` | array | Optional exclusive list of platforms that include this case | @@ -458,7 +465,9 @@ SDK sends, or the reason the SDK drops it. Validate them locally with `python3 ci/validate-flag-selection.py`. See [flag-selection/README.md](flag-selection/README.md) for the semantics and -how to run the cases against each request form. +how to run the cases against each request form. The +[flag selection suite](#flag-selection-suite) runs selections through an SDK +end to end. ## Automated Validation @@ -468,6 +477,8 @@ object keys, malformed evaluation-case envelopes, mismatched flag or variation map keys, and references to unknown flags without a `FLAG_NOT_FOUND` expectation. Capability expectations may instead use `INVALID_CONTEXT` when context validation precedes flag lookup. The baseline still requires `FLAG_NOT_FOUND`. +In suites, it also checks that `flag-tags.json` names only the suite's flags, the +shape of the flag selection case fields, and that they are the same across a file. The validator deliberately does not fully schema-check allocation internals. Some fixtures contain malformed flag fields on purpose to verify that consumers @@ -578,6 +589,41 @@ and `cases/` directory. | Suite | Description | |-------|-------------| | `dependent-flags` | Direct and transitive dependencies, short-circuiting, repeated prerequisites, failure propagation, depth limiting, and cycles | +| `flag-selection` | SDK flag selection end to end: tags, key patterns, both, canonical form, a tag with interior whitespace, an empty selection, selections the SDK drops, and a selection the server ignores. Needs a server double; see below | + +### Flag selection suite + +`evaluation-suites/flag-selection/` checks that an SDK sends its flag selection +on the agentless configuration request and serves only the flags the server +returns. Each case file is one provider start, so every case in a file carries +the same `providerOptions`, `expectations.configurationRequest`, and +`configurationResponse`; the validator enforces this. + +The runner cannot load the suite's `ufc-config.json` directly. It serves it +through a server double that applies the selection in the request with the +[flag selection semantics](flag-selection/README.md) and the tags in +`flag-tags.json` (a flag that is not listed has no tags), and returns only the +selected flags. For each case file: + +1. Configure the server double. When `configurationResponse.flagSelectionIgnored` + is set, it answers a valid selection with every flag and that reason in the + `DD-Flag-Selection-Ignored` header. +2. Start the provider in agentless mode against the server double, passing + `providerOptions.flagSelection` untouched to the SDK's flag selection option. + Omit the option when the field is absent. +3. Check the first configuration request: it carries each + `expectations.configurationRequest` value as a repeated `selected_flag_tags` + or `selected_flag_key_patterns` parameter, in order, or no selection + parameter for `{"flagSelection": null}`. +4. Evaluate the applicable cases as usual. A flag the server omitted evaluates + to the caller default with `ERROR` / `FLAG_NOT_FOUND`. + +An SDK sends no selection when the selection is empty after canonicalization, +breaks a limit in [flag-selection/README.md](flag-selection/README.md#semantics), +or would make the percent-encoded request URL longer than 8192 bytes +([Fastly's URL limit](https://docs.fastly.com/products/network-services-resource-limits)). +The long-URL case uses non-ASCII tags, so the URL is over that limit only once +percent-encoded. ## Origin diff --git a/ci/test_validate_fixtures.py b/ci/test_validate_fixtures.py index c5a9064f..0d084abf 100644 --- a/ci/test_validate_fixtures.py +++ b/ci/test_validate_fixtures.py @@ -168,6 +168,128 @@ def test_self_contained_suite_rejects_missing_or_extra_entries(self) -> None: validator._validate_suite(config_path, cases, self.platforms, self_contained=True) +class FlagSelectionSuiteValidationTest(unittest.TestCase): + platforms = frozenset({"php", "python"}) + path = validator.EVALUATION_SUITES_DIRECTORY / "example" / "cases" / "test-case.json" + + @staticmethod + def case(**fields: object) -> dict[str, object]: + return { + **EvaluationSuiteValidationTest.case("suite-flag"), + "providerOptions": {"flagSelection": {"tags": [" team:checkout "]}}, + "expectations": { + "configurationRequest": {"selectedFlagTags": ["team:checkout"], "selectedFlagKeyPatterns": []} + }, + **fields, + } + + def validate_case(self, case: dict[str, object]) -> None: + validator._validate_evaluation_case(self.path, 0, case, {"suite-flag": {}}, self.platforms) + + def write_suite(self, directory: str, flag_tags: object, cases: list[dict[str, object]]) -> tuple[Path, Path]: + suite = Path(directory) / "example" + (suite / "cases").mkdir(parents=True) + config_path = suite / "ufc-config.json" + config_path.write_text(json.dumps(EvaluationSuiteValidationTest.config("suite-flag")), encoding="utf-8") + (suite / "flag-tags.json").write_text(json.dumps(flag_tags), encoding="utf-8") + (suite / "cases" / "test-case.json").write_text(json.dumps(cases), encoding="utf-8") + return config_path, suite / "cases" + + def test_suite_accepts_flag_tags_for_its_own_flags(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + config_path, cases = self.write_suite( + temporary_directory, {"flags": [{"key": "suite-flag", "tags": ["team:checkout"]}]}, [self.case()] + ) + self.assertEqual( + (1, 1), validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + ) + + (config_path.parent / "unexpected.json").write_text("{}", encoding="utf-8") + with self.assertRaisesRegex(validator.FixtureValidationError, "unexpected suite entry"): + validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + + def test_flag_tags_reject_invalid_shapes_and_unknown_flags(self) -> None: + invalid = ( + ({"flags": [{"key": "root-only-flag", "tags": []}]}, "must name a flag"), + ({"flags": [{"key": "suite-flag", "tags": []}, {"key": "suite-flag", "tags": []}]}, "is repeated"), + ({"flags": [{"key": "suite-flag", "tags": ["", "team:checkout"]}]}, "non-empty strings"), + ({"flags": [{"key": "suite-flag"}]}, "only key and tags"), + ({"flags": []}, "non-empty array"), + ({"flags": [], "extra": True}, "only flags"), + ) + for flag_tags, message in invalid: + with self.subTest(flag_tags=flag_tags), tempfile.TemporaryDirectory() as temporary_directory: + config_path, cases = self.write_suite(temporary_directory, flag_tags, [self.case()]) + with self.assertRaisesRegex(validator.FixtureValidationError, message): + validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + + def test_provider_start_fields_accept_valid_shapes(self) -> None: + self.validate_case(self.case()) + self.validate_case(self.case(configurationResponse={"flagSelectionIgnored": "tag_load_failed"})) + self.validate_case( + self.case( + providerOptions={"flagSelection": {"tags": [], "keyPatterns": []}}, + expectations={"configurationRequest": {"flagSelection": None}}, + ) + ) + + def test_provider_start_fields_reject_invalid_shapes(self) -> None: + no_selection = {"configurationRequest": {"flagSelection": None}} + invalid = ( + ({"providerOptions": []}, "providerOptions must be an object"), + ({"providerOptions": {"flagSelections": {}}}, "unknown fields: flagSelections"), + ({"providerOptions": {"flagSelection": {"tags": [1]}}}, "string arrays"), + ({"providerOptions": {"flagSelection": {"labels": []}}}, "string arrays"), + ({"expectations": {}}, "requires expectations.configurationRequest"), + ({"expectations": {"configurationRequest": {"flagSelection": False}}}, "flagSelection\": null"), + ( + { + "expectations": { + "configurationRequest": {"selectedFlagTags": ["b", "a"], "selectedFlagKeyPatterns": []} + } + }, + "sorted by UTF-8 bytes", + ), + ( + { + "expectations": { + "configurationRequest": {"selectedFlagTags": ["a", "a"], "selectedFlagKeyPatterns": []} + } + }, + "without duplicates", + ), + ( + {"expectations": {"configurationRequest": {"selectedFlagTags": [], "selectedFlagKeyPatterns": []}}}, + "sends no selection", + ), + ({"configurationResponse": {"flagSelectionIgnored": "unknown_reason"}}, "must be one of"), + ({"configurationResponse": {"reason": "tag_load_failed"}}, "only flagSelectionIgnored"), + ( + {"expectations": no_selection, "configurationResponse": {"flagSelectionIgnored": "tag_load_failed"}}, + "sends a selection", + ), + ) + for fields, message in invalid: + with self.subTest(fields=fields): + with self.assertRaisesRegex(validator.FixtureValidationError, message): + self.validate_case(self.case(**fields)) + + def test_cases_in_a_file_share_one_provider_start(self) -> None: + other_start = self.case( + providerOptions={"flagSelection": {"tags": ["team:search"]}}, + expectations={ + "configurationRequest": {"selectedFlagTags": ["team:search"], "selectedFlagKeyPatterns": []} + }, + ) + for cases in ([self.case(), other_start], [self.case(), EvaluationSuiteValidationTest.case("suite-flag")]): + with self.subTest(cases=cases), tempfile.TemporaryDirectory() as temporary_directory: + config_path, cases_directory = self.write_suite( + temporary_directory, {"flags": [{"key": "suite-flag", "tags": []}]}, cases + ) + with self.assertRaisesRegex(validator.FixtureValidationError, "case 1 .* must equal case 0's"): + validator._validate_suite(config_path, cases_directory, self.platforms, self_contained=True) + + class EvaluationPlatformValidationTest(unittest.TestCase): platforms = frozenset({"php", "python"}) path = validator.EVALUATION_CASES_DIRECTORY / "test-case-filter.json" diff --git a/ci/validate-fixtures.py b/ci/validate-fixtures.py index 6c290e1f..4e359084 100755 --- a/ci/validate-fixtures.py +++ b/ci/validate-fixtures.py @@ -20,6 +20,13 @@ ) VALID_VARIATION_TYPES = frozenset({"BOOLEAN", "INTEGER", "JSON", "NUMERIC", "STRING"}) FEATURE_DESTINATIONS = frozenset({"HOOK", "EXPOSURE", "EVALUATION"}) +SUITE_FLAG_TAGS_FILE = "flag-tags.json" +FLAG_SELECTION_FIELDS = frozenset({"tags", "keyPatterns"}) +CANONICAL_SELECTION_FIELDS = ("selectedFlagTags", "selectedFlagKeyPatterns") +NO_SELECTION_REQUEST = {"flagSelection": None} +FLAG_SELECTION_IGNORED_REASONS = frozenset( + {"malformed_selection", "too_many_values", "invalid_key_pattern", "invalid_tag", "tag_load_failed"} +) REQUIRED_CASE_FIELDS = frozenset( { "attributes", @@ -192,6 +199,108 @@ def _validate_evaluation_case( path, f"{location} capabilities.{capability}", result, flag, configured_flags, allow_invalid_context=True, ) + _validate_provider_start(path, location, value) + + +def _is_string_list(value: object) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _validate_provider_start(path: Path, location: str, case: dict[str, object]) -> None: + options = case.get("providerOptions", {}) + if not isinstance(options, dict): + _fail(path, f"{location} providerOptions must be an object") + unknown = sorted(set(options).difference({"flagSelection"})) + if unknown: + _fail(path, f"{location} providerOptions has unknown fields: {', '.join(unknown)}") + if "flagSelection" in options: + selection = options["flagSelection"] + if ( + not isinstance(selection, dict) + or not set(selection).issubset(FLAG_SELECTION_FIELDS) + or not all(_is_string_list(values) for values in selection.values()) + ): + _fail( + path, + f"{location} providerOptions.flagSelection must be an object with optional tags and keyPatterns " + "string arrays", + ) + + expectations = case.get("expectations", {}) + if not isinstance(expectations, dict): + _fail(path, f"{location} expectations must be an object") + sends_selection = False + if "configurationRequest" in expectations: + sends_selection = _validate_configuration_request( + path, f"{location} expectations.configurationRequest", expectations["configurationRequest"] + ) + elif "flagSelection" in options: + _fail(path, f"{location} providerOptions.flagSelection requires expectations.configurationRequest") + + if "configurationResponse" in case: + response = case["configurationResponse"] + if not isinstance(response, dict) or set(response) != {"flagSelectionIgnored"}: + _fail(path, f"{location} configurationResponse must be an object with only flagSelectionIgnored") + if response["flagSelectionIgnored"] not in FLAG_SELECTION_IGNORED_REASONS: + _fail( + path, + f"{location} configurationResponse.flagSelectionIgnored must be one of " + f"{', '.join(sorted(FLAG_SELECTION_IGNORED_REASONS))}", + ) + if not sends_selection: + _fail(path, f"{location} configurationResponse requires a configurationRequest that sends a selection") + + +def _validate_configuration_request(path: Path, location: str, request: object) -> bool: + """Validate the canonical selection on the configuration request and return whether one is sent.""" + if request == NO_SELECTION_REQUEST: + return False + if not isinstance(request, dict) or set(request) != set(CANONICAL_SELECTION_FIELDS): + _fail( + path, + f"{location} must be {{\"flagSelection\": null}} or hold only selectedFlagTags and " + "selectedFlagKeyPatterns", + ) + for field in CANONICAL_SELECTION_FIELDS: + values = request[field] + if ( + not _is_string_list(values) + or not all(values) + or values != sorted(set(values), key=lambda value: value.encode("utf-8")) + ): + _fail(path, f"{location}.{field} must hold non-empty strings sorted by UTF-8 bytes without duplicates") + if not any(request[field] for field in CANONICAL_SELECTION_FIELDS): + _fail(path, f"{location} sends no selection; use {{\"flagSelection\": null}}") + return True + + +def _provider_start(case: dict[str, object]) -> tuple[object, object, object]: + expectations = case.get("expectations", {}) + return case.get("providerOptions"), expectations.get("configurationRequest"), case.get("configurationResponse") + + +def _validate_flag_tags(path: Path, configured_flags: dict[str, object]) -> None: + if path.is_symlink() or not path.is_file(): + _fail(path, "must be a regular file, not a symlink") + value = _load_json(path) + if not isinstance(value, dict) or set(value) != {"flags"}: + _fail(path, "top-level value must be an object with only flags") + flags = value["flags"] + if not isinstance(flags, list) or not flags: + _fail(path, "flags must be a non-empty array") + keys: set[str] = set() + for index, entry in enumerate(flags): + location = f"flags[{index}]" + if not isinstance(entry, dict) or set(entry) != {"key", "tags"}: + _fail(path, f"{location} must be an object with only key and tags") + key = entry["key"] + if not isinstance(key, str) or key not in configured_flags: + _fail(path, f"{location}.key must name a flag in the suite's ufc-config.json") + if key in keys: + _fail(path, f"{location}.key {key!r} is repeated") + keys.add(key) + if not _is_string_list(entry["tags"]) or not all(entry["tags"]): + _fail(path, f"{location}.tags must be an array of non-empty strings") def _validate_result( @@ -255,12 +364,15 @@ def _validate_suite( if self_contained: suite_directory = config_path.parent - allowed_entries = {config_path.name, cases_directory.name} + allowed_entries = {config_path.name, cases_directory.name, SUITE_FLAG_TAGS_FILE} for entry in suite_directory.iterdir(): if entry.name not in allowed_entries: - _fail(entry, "unexpected suite entry; only ufc-config.json and cases/ are allowed") + _fail(entry, "unexpected suite entry; only ufc-config.json, flag-tags.json, and cases/ are allowed") configured_flags = _validate_config(config_path, _load_json(config_path)) + flag_tags_path = config_path.parent / SUITE_FLAG_TAGS_FILE + if self_contained and (flag_tags_path.exists() or flag_tags_path.is_symlink()): + _validate_flag_tags(flag_tags_path, configured_flags) evaluation_paths: list[Path] = [] for entry in sorted(cases_directory.iterdir()): if entry.is_symlink() or not entry.is_file(): @@ -281,6 +393,13 @@ def _validate_suite( for index, case in enumerate(cases): _validate_evaluation_case(path, index, case, configured_flags, platforms) + # A case file is one provider start, so its cases share the start's inputs and request. + if _provider_start(case) != _provider_start(cases[0]): + _fail( + path, + f"case {index} providerOptions, expectations.configurationRequest, and configurationResponse " + "must equal case 0's", + ) case_count += 1 return case_count, len(evaluation_paths) diff --git a/evaluation-suites/flag-selection/cases/canonical-form.json b/evaluation-suites/flag-selection/cases/canonical-form.json new file mode 100644 index 00000000..d28fb87d --- /dev/null +++ b/evaluation-suites/flag-selection/cases/canonical-form.json @@ -0,0 +1,148 @@ +[ + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by tag team:search.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by key pattern ops-*.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/empty-selection.json b/evaluation-suites/flag-selection/cases/empty-selection.json new file mode 100644 index 00000000..ecbef12e --- /dev/null +++ b/evaluation-suites/flag-selection/cases/empty-selection.json @@ -0,0 +1,140 @@ +[ + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/interior-whitespace-tag.json b/evaluation-suites/flag-selection/cases/interior-whitespace-tag.json new file mode 100644 index 00000000..4abf3f66 --- /dev/null +++ b/evaluation-suites/flag-selection/cases/interior-whitespace-tag.json @@ -0,0 +1,145 @@ +[ + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by tag release:summer sale.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/key-patterns.json b/evaluation-suites/flag-selection/cases/key-patterns.json new file mode 100644 index 00000000..deeffdfe --- /dev/null +++ b/evaluation-suites/flag-selection/cases/key-patterns.json @@ -0,0 +1,144 @@ +[ + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by key pattern search-*.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by key pattern *-sale.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/sdk-drops-long-url.json b/evaluation-suites/flag-selection/cases/sdk-drops-long-url.json new file mode 100644 index 00000000..e80b90c8 --- /dev/null +++ b/evaluation-suites/flag-selection/cases/sdk-drops-long-url.json @@ -0,0 +1,46 @@ +[ + { + "description": "Percent-encoded, the selection takes the request URL over 8192 bytes, so the SDK sends none and every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "long-01:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-02:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-03:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-04:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-05:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-06:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-07:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-08:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-09:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-10:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-11:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-12:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-13:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-14:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-15:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-16:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-17:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-18:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-19:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-20:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-21:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-22:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-23:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-24:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-25:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-26:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-27:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-28:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-29:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-30:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Percent-encoded, the selection takes the request URL over 8192 bytes, so the SDK sends none and every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "long-01:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-02:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-03:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-04:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-05:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-06:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-07:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-08:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-09:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-10:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-11:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-12:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-13:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-14:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-15:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-16:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-17:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-18:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-19:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-20:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-21:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-22:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-23:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-24:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-25:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-26:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-27:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-28:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-29:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-30:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/sdk-drops-too-many-values.json b/evaluation-suites/flag-selection/cases/sdk-drops-too-many-values.json new file mode 100644 index 00000000..90399c3f --- /dev/null +++ b/evaluation-suites/flag-selection/cases/sdk-drops-too-many-values.json @@ -0,0 +1,134 @@ +[ + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/server-ignores-selection.json b/evaluation-suites/flag-selection/cases/server-ignores-selection.json new file mode 100644 index 00000000..e800ffda --- /dev/null +++ b/evaluation-suites/flag-selection/cases/server-ignores-selection.json @@ -0,0 +1,158 @@ +[ + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/tags-or-key-patterns.json b/evaluation-suites/flag-selection/cases/tags-or-key-patterns.json new file mode 100644 index 00000000..1af0831f --- /dev/null +++ b/evaluation-suites/flag-selection/cases/tags-or-key-patterns.json @@ -0,0 +1,150 @@ +[ + { + "description": "Selected by key pattern checkout-b*.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by tag team:ops.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/tags.json b/evaluation-suites/flag-selection/cases/tags.json new file mode 100644 index 00000000..f0d7f81b --- /dev/null +++ b/evaluation-suites/flag-selection/cases/tags.json @@ -0,0 +1,144 @@ +[ + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/flag-tags.json b/evaluation-suites/flag-selection/flag-tags.json new file mode 100644 index 00000000..fb041769 --- /dev/null +++ b/evaluation-suites/flag-selection/flag-tags.json @@ -0,0 +1,10 @@ +{ + "flags": [ + {"key": "checkout-banner", "tags": ["team:checkout"]}, + {"key": "checkout-pricing", "tags": ["team:checkout"]}, + {"key": "search-ranking", "tags": ["team:search"]}, + {"key": "summer-sale", "tags": ["release:summer sale"]}, + {"key": "ops-kill-switch", "tags": ["team:ops"]}, + {"key": "untagged-flag", "tags": []} + ] +} diff --git a/evaluation-suites/flag-selection/ufc-config.json b/evaluation-suites/flag-selection/ufc-config.json new file mode 100644 index 00000000..22c5b154 --- /dev/null +++ b/evaluation-suites/flag-selection/ufc-config.json @@ -0,0 +1,177 @@ +{ + "createdAt": "2026-11-03T00:00:00.000Z", + "format": "SERVER", + "environment": { + "name": "Test" + }, + "flags": { + "checkout-banner": { + "key": "checkout-banner", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "checkout-pricing": { + "key": "checkout-pricing", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "search-ranking": { + "key": "search-ranking", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "summer-sale": { + "key": "summer-sale", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "ops-kill-switch": { + "key": "ops-kill-switch", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "untagged-flag": { + "key": "untagged-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + } + } +} diff --git a/flag-selection/README.md b/flag-selection/README.md index 79278b2d..39237bcf 100644 --- a/flag-selection/README.md +++ b/flag-selection/README.md @@ -160,6 +160,13 @@ omitted list. Then: When both are empty, the SDK sends no selection parameters. - `sdk.dropped`: the SDK drops the selection with this reason and sends none. +## End-to-end suite + +[`evaluation-suites/flag-selection/`](../evaluation-suites/flag-selection/) +runs selections through an SDK against a server double that applies these +semantics to that suite's `ufc-config.json` and `flag-tags.json`. See +[flag selection suite](../README.md#flag-selection-suite). + ## Consumers Each consumer pins a merged commit of this repository. A change to these