From 7ba049d75348a88f50a5fdd45d4164dd211aa8f1 Mon Sep 17 00:00:00 2001 From: Mark Cai Date: Mon, 2 Nov 2026 18:54:05 +0000 Subject: [PATCH 1/3] Add shared tags for ufc-config.json flags Mock configuration endpoints that serve ufc-config.json need flag tags to apply a flag selection. flag-selection/ufc-config-tags.json lists them in the vectors' flags shape, and the validator checks that every key exists in ufc-config.json. --- .github/workflows/validate-flag-selection.yml | 1 + README.md | 7 ++- ci/validate-flag-selection.py | 52 +++++++++++++------ flag-selection/README.md | 21 +++++++- flag-selection/ufc-config-tags.json | 7 +++ 5 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 flag-selection/ufc-config-tags.json diff --git a/.github/workflows/validate-flag-selection.yml b/.github/workflows/validate-flag-selection.yml index 185e6f92..751afbb9 100644 --- a/.github/workflows/validate-flag-selection.yml +++ b/.github/workflows/validate-flag-selection.yml @@ -6,6 +6,7 @@ on: - .github/workflows/validate-flag-selection.yml - ci/validate-flag-selection.py - flag-selection/** + - ufc-config.json permissions: contents: read diff --git a/README.md b/README.md index 3167b562..903e589b 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,8 @@ ffe-system-test-data/ │ └── cases/ # Precomputed assignment fixtures, discovered recursively ├── flag-selection/ │ ├── README.md # Flag selection semantics and vector format -│ └── selection-vectors.json # Server and SDK flag selection test vectors +│ ├── selection-vectors.json # Server and SDK flag selection test vectors +│ └── ufc-config-tags.json # Tags of ufc-config.json flags, for selection against that configuration ├── schemas/ # Precomputed assignment fixture schema └── ci/ # Static validators and downstream comparison runner ``` @@ -451,6 +452,10 @@ ignored. Cases with an optional `sdk` field also give the canonical selection an SDK sends, or the reason the SDK drops it. Validate them locally with `python3 ci/validate-flag-selection.py`. +`flag-selection/ufc-config-tags.json` gives tags to some `ufc-config.json` +flags, so a mock configuration endpoint can apply a selection to that +configuration. + See [flag-selection/README.md](flag-selection/README.md) for the semantics and how to run the cases against each request form. diff --git a/ci/validate-flag-selection.py b/ci/validate-flag-selection.py index fb87c210..dd9c4b36 100644 --- a/ci/validate-flag-selection.py +++ b/ci/validate-flag-selection.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Validate the static shape of the flag selection vectors.""" +"""Validate the static shape of the flag selection vectors and the shared UFC tags.""" from __future__ import annotations @@ -10,6 +10,8 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] VECTORS_PATH = REPOSITORY_ROOT / "flag-selection" / "selection-vectors.json" +UFC_TAGS_PATH = REPOSITORY_ROOT / "flag-selection" / "ufc-config-tags.json" +UFC_CONFIG_PATH = REPOSITORY_ROOT / "ufc-config.json" QUERY_PARAMETERS = frozenset({"selected_flag_tags", "selected_flag_key_patterns"}) IGNORED_REASONS = frozenset({"malformed_selection", "too_many_values", "invalid_key_pattern", "invalid_tag"}) @@ -18,8 +20,8 @@ class VectorValidationError(ValueError): """The vectors file is valid JSON but violates the documented format.""" -def _fail(message: str) -> NoReturn: - raise VectorValidationError(f"{VECTORS_PATH.relative_to(REPOSITORY_ROOT)}: {message}") +def _fail(message: str, path: Path = VECTORS_PATH) -> NoReturn: + raise VectorValidationError(f"{path.relative_to(REPOSITORY_ROOT)}: {message}") def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: @@ -31,47 +33,62 @@ def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object] return result -def _load() -> object: +def _load(path: Path = VECTORS_PATH) -> object: try: - return json.loads(VECTORS_PATH.read_text(encoding="utf-8"), object_pairs_hook=_reject_duplicate_keys) + return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=_reject_duplicate_keys) except json.JSONDecodeError as error: - _fail(f"invalid JSON at line {error.lineno}, column {error.colno}: {error.msg}") + _fail(f"invalid JSON at line {error.lineno}, column {error.colno}: {error.msg}", path) except VectorValidationError as error: - _fail(str(error)) + _fail(str(error), path) def _is_string_list(value: object) -> bool: return isinstance(value, list) and all(isinstance(item, str) for item in value) -def _check_keys(location: str, value: dict[str, object], required: set[str], optional: set[str]) -> None: +def _check_keys( + location: str, value: dict[str, object], required: set[str], optional: set[str], path: Path = VECTORS_PATH +) -> None: missing = sorted(required.difference(value)) if missing: - _fail(f"{location} is missing {', '.join(missing)}") + _fail(f"{location} is missing {', '.join(missing)}", path) unknown = sorted(set(value).difference(required, optional)) if unknown: - _fail(f"{location} has unknown fields {', '.join(unknown)}") + _fail(f"{location} has unknown fields {', '.join(unknown)}", path) -def _validate_flags(value: object) -> set[str]: +def _validate_flags(value: object, path: Path = VECTORS_PATH) -> set[str]: if not isinstance(value, list) or not value: - _fail("flags must be a non-empty array") + _fail("flags must be a non-empty array", path) keys: set[str] = set() for index, flag in enumerate(value): location = f"flags[{index}]" if not isinstance(flag, dict): - _fail(f"{location} must be an object") - _check_keys(location, flag, {"key", "tags"}, set()) + _fail(f"{location} must be an object", path) + _check_keys(location, flag, {"key", "tags"}, set(), path) if not isinstance(flag["key"], str): - _fail(f"{location}.key must be a string") + _fail(f"{location}.key must be a string", path) if flag["key"] in keys: - _fail(f"{location}.key {flag['key']!r} is repeated") + _fail(f"{location}.key {flag['key']!r} is repeated", path) if not _is_string_list(flag["tags"]): - _fail(f"{location}.tags must be an array of strings") + _fail(f"{location}.tags must be an array of strings", path) keys.add(flag["key"]) return keys +def _validate_ufc_tags() -> int: + tags = _load(UFC_TAGS_PATH) + if not isinstance(tags, dict): + _fail("top-level value must be an object", UFC_TAGS_PATH) + _check_keys("top-level object", tags, {"flags"}, set(), UFC_TAGS_PATH) + tagged_keys = _validate_flags(tags["flags"], UFC_TAGS_PATH) + config = json.loads(UFC_CONFIG_PATH.read_text(encoding="utf-8")) + unknown = sorted(tagged_keys.difference(config["flags"])) + if unknown: + _fail(f"flags names keys missing from ufc-config.json: {', '.join(unknown)}", UFC_TAGS_PATH) + return len(tagged_keys) + + def _validate_case(index: int, case: object, flag_keys: set[str]) -> str: location = f"cases[{index}]" if not isinstance(case, dict): @@ -165,6 +182,7 @@ def main() -> None: names.add(name) print(f"Validated {len(cases)} flag selection vectors against {len(flag_keys)} flags.") + print(f"Validated tags for {_validate_ufc_tags()} ufc-config.json flags.") if __name__ == "__main__": diff --git a/flag-selection/README.md b/flag-selection/README.md index 79278b2d..73c8e6c9 100644 --- a/flag-selection/README.md +++ b/flag-selection/README.md @@ -160,6 +160,24 @@ omitted list. Then: When both are empty, the SDK sends no selection parameters. - `sdk.dropped`: the SDK drops the selection with this reason and sends none. +## Tags for `ufc-config.json` + +UFC responses carry no tags, so a mock configuration endpoint that serves the +shared [`ufc-config.json`](../ufc-config.json) cannot apply a tag selection on +its own. `ufc-config-tags.json` lists the tags of some of its flags, in the same +shape as `flags` above. A flag that is not listed has no tags. + +```json +{ + "flags": [{ "key": "basic-flag", "tags": ["team:checkout", "release:summer sale"] }] +} +``` + +A mock that serves `ufc-config.json` builds the flag set from every key in +`ufc-config.json` and the tags listed here, applies the selection with the +semantics above, and returns only the selected entries of `flags`. The vectors +do not run against this flag set. + ## Consumers Each consumer pins a merged commit of this repository. A change to these @@ -178,4 +196,5 @@ distinct, and present in `flags`. For `sdk`, it checks that every `query` case has one and no `rawQuery` case does, that `sdk.selection` lists hold non-empty strings sorted by UTF-8 bytes without duplicates, that `sdk.selection` appears only on `applied` cases, and that `sdk.dropped` equals `expected.reason`. It -does not recompute expected outcomes. +does not recompute expected outcomes. For `ufc-config-tags.json`, it checks the +same `flags` shape and that every key names a flag in `ufc-config.json`. diff --git a/flag-selection/ufc-config-tags.json b/flag-selection/ufc-config-tags.json new file mode 100644 index 00000000..a88b7453 --- /dev/null +++ b/flag-selection/ufc-config-tags.json @@ -0,0 +1,7 @@ +{ + "flags": [ + {"key": "basic-flag", "tags": ["team:checkout", "release:summer sale"]}, + {"key": "numeric_flag", "tags": ["team:checkout"]}, + {"key": "experiment-flag", "tags": ["team:search"]} + ] +} From 935b847ae77a3640eebbaf27ae6dc12d53242ced Mon Sep 17 00:00:00 2001 From: Mark Cai Date: Mon, 2 Nov 2026 19:50:34 +0000 Subject: [PATCH 2/3] Describe validation errors for both flag selection files --- ci/validate-flag-selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/validate-flag-selection.py b/ci/validate-flag-selection.py index dd9c4b36..024cc03b 100644 --- a/ci/validate-flag-selection.py +++ b/ci/validate-flag-selection.py @@ -17,7 +17,7 @@ class VectorValidationError(ValueError): - """The vectors file is valid JSON but violates the documented format.""" + """A flag selection file is valid JSON but violates the documented format.""" def _fail(message: str, path: Path = VECTORS_PATH) -> NoReturn: From be2ab8372d9cce5c77fbb940266564e7b86ce83a Mon Sep 17 00:00:00 2001 From: Mark Cai Date: Tue, 3 Nov 2026 21:40:06 +0000 Subject: [PATCH 3/3] Add the flag-selection evaluation suite An SDK given a flag selection must send it in canonical form on the agentless configuration request and serve only the flags the server returns. evaluation-suites/flag-selection checks that end to end: a small UFC, flag-tags.json with the tags UFC does not carry, and one selection per case file covering tags, key patterns, both, canonical form, a tag with interior whitespace, an empty selection, two SDK drops and a selection the server ignores. Cases gain providerOptions.flagSelection, expectations.configurationRequest and configurationResponse.flagSelectionIgnored. The validator checks their shapes, that all cases in a file share them, and that flag-tags.json names only the suite's flags. It accepts flag-tags.json as a third suite entry. This replaces the root-level flag-selection/ufc-config-tags.json; the shared selection-vectors.json is unchanged. --- .github/workflows/validate-flag-selection.yml | 1 - CONTRIBUTING.md | 2 + README.md | 61 +++++- ci/test_validate_fixtures.py | 122 ++++++++++++ ci/validate-fixtures.py | 123 +++++++++++- ci/validate-flag-selection.py | 54 ++---- .../flag-selection/cases/canonical-form.json | 148 +++++++++++++++ .../flag-selection/cases/empty-selection.json | 140 ++++++++++++++ .../cases/interior-whitespace-tag.json | 145 ++++++++++++++ .../flag-selection/cases/key-patterns.json | 144 ++++++++++++++ .../cases/sdk-drops-long-url.json | 46 +++++ .../cases/sdk-drops-too-many-values.json | 134 +++++++++++++ .../cases/server-ignores-selection.json | 158 ++++++++++++++++ .../cases/tags-or-key-patterns.json | 150 +++++++++++++++ .../flag-selection/cases/tags.json | 144 ++++++++++++++ .../flag-selection/flag-tags.json | 10 + .../flag-selection/ufc-config.json | 177 ++++++++++++++++++ flag-selection/README.md | 24 +-- flag-selection/ufc-config-tags.json | 7 - 19 files changed, 1716 insertions(+), 74 deletions(-) create mode 100644 evaluation-suites/flag-selection/cases/canonical-form.json create mode 100644 evaluation-suites/flag-selection/cases/empty-selection.json create mode 100644 evaluation-suites/flag-selection/cases/interior-whitespace-tag.json create mode 100644 evaluation-suites/flag-selection/cases/key-patterns.json create mode 100644 evaluation-suites/flag-selection/cases/sdk-drops-long-url.json create mode 100644 evaluation-suites/flag-selection/cases/sdk-drops-too-many-values.json create mode 100644 evaluation-suites/flag-selection/cases/server-ignores-selection.json create mode 100644 evaluation-suites/flag-selection/cases/tags-or-key-patterns.json create mode 100644 evaluation-suites/flag-selection/cases/tags.json create mode 100644 evaluation-suites/flag-selection/flag-tags.json create mode 100644 evaluation-suites/flag-selection/ufc-config.json delete mode 100644 flag-selection/ufc-config-tags.json diff --git a/.github/workflows/validate-flag-selection.yml b/.github/workflows/validate-flag-selection.yml index 751afbb9..185e6f92 100644 --- a/.github/workflows/validate-flag-selection.yml +++ b/.github/workflows/validate-flag-selection.yml @@ -6,7 +6,6 @@ on: - .github/workflows/validate-flag-selection.yml - ci/validate-flag-selection.py - flag-selection/** - - ufc-config.json permissions: contents: read diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3af4038..336e7a23 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,8 @@ When a new feature's UFC syntax itself is not supported by every SDK, create a self-contained `evaluation-suites//` directory containing its own `ufc-config.json` and `cases/`. Do not add that syntax to the legacy root config: a suite the consumer has not enabled must not have its configuration parsed. +A suite may also hold `flag-tags.json` when its server double needs flag tags, +as `flag-selection` does; see [flag selection suite](README.md#flag-selection-suite). For selected split/allocation metadata, add optional `result.serialId` (integer or `null`) and `result.doLog` (boolean) expectations. A serial ID of `0` is valid; diff --git a/README.md b/README.md index 79cd80b0..3a6df2ca 100644 --- a/README.md +++ b/README.md @@ -87,16 +87,19 @@ ffe-system-test-data/ ├── evaluation-cases/ │ └── test-*.json # Evaluation test case files ├── evaluation-suites/ # Optional self-contained, explicitly activated feature suites -│ └── dependent-flags/ -│ ├── ufc-config.json # UFC containing only dependent-flag fixtures -│ └── cases/ # Evaluation cases for this feature +│ ├── dependent-flags/ +│ │ ├── ufc-config.json # UFC containing only dependent-flag fixtures +│ │ └── cases/ # Evaluation cases for this feature +│ └── flag-selection/ +│ ├── ufc-config.json # Flags a server double narrows to the SDK's flag selection +│ ├── flag-tags.json # Tags of those flags, which UFC does not carry +│ └── cases/ # One flag selection per case file ├── precomputed-assignments/ │ ├── README.md # Precomputed assignment fixture format │ └── cases/ # Precomputed assignment fixtures, discovered recursively ├── flag-selection/ │ ├── README.md # Flag selection semantics and vector format -│ ├── selection-vectors.json # Server and SDK flag selection test vectors -│ └── ufc-config-tags.json # Tags of ufc-config.json flags, for selection against that configuration +│ └── selection-vectors.json # Server and SDK flag selection test vectors ├── schemas/ # Precomputed assignment fixture schema └── ci/ # Static validators and downstream comparison runner ``` @@ -231,6 +234,9 @@ Each evaluation case uses a universal schema with the following fields: | `result.doLog` | boolean | Optional expected selected allocation's logging decision | | `result.features` | object | Optional exact feature maps for `HOOK`, `EXPOSURE`, and `EVALUATION`; empty maps assert no features, omission leaves routing unspecified | | `capabilities` | object | Optional expected results when a named capability is enabled; uses the same result format | +| `providerOptions.flagSelection` | object | Optional raw flag selection, `{"tags": [...], "keyPatterns": [...]}`, passed untouched to the SDK's flag selection provider option. See [flag selection suite](#flag-selection-suite) | +| `expectations.configurationRequest` | object | Optional selection the SDK sends on its configuration request: canonical `selectedFlagTags` and `selectedFlagKeyPatterns` lists, or `{"flagSelection": null}` when it sends none | +| `configurationResponse.flagSelectionIgnored` | string | Optional reason the server double gives for ignoring the valid selection it receives | | `_skip` | array | Optional platforms that exclude this case | | `_include` | array | Optional exclusive list of platforms that include this case | @@ -458,12 +464,10 @@ ignored. Cases with an optional `sdk` field also give the canonical selection an SDK sends, or the reason the SDK drops it. Validate them locally with `python3 ci/validate-flag-selection.py`. -`flag-selection/ufc-config-tags.json` gives tags to some `ufc-config.json` -flags, so a mock configuration endpoint can apply a selection to that -configuration. - See [flag-selection/README.md](flag-selection/README.md) for the semantics and -how to run the cases against each request form. +how to run the cases against each request form. The +[flag selection suite](#flag-selection-suite) runs selections through an SDK +end to end. ## Automated Validation @@ -473,6 +477,8 @@ object keys, malformed evaluation-case envelopes, mismatched flag or variation map keys, and references to unknown flags without a `FLAG_NOT_FOUND` expectation. Capability expectations may instead use `INVALID_CONTEXT` when context validation precedes flag lookup. The baseline still requires `FLAG_NOT_FOUND`. +In suites, it also checks that `flag-tags.json` names only the suite's flags, the +shape of the flag selection case fields, and that they are the same across a file. The validator deliberately does not fully schema-check allocation internals. Some fixtures contain malformed flag fields on purpose to verify that consumers @@ -583,6 +589,41 @@ and `cases/` directory. | Suite | Description | |-------|-------------| | `dependent-flags` | Direct and transitive dependencies, short-circuiting, repeated prerequisites, failure propagation, depth limiting, and cycles | +| `flag-selection` | SDK flag selection end to end: tags, key patterns, both, canonical form, a tag with interior whitespace, an empty selection, selections the SDK drops, and a selection the server ignores. Needs a server double; see below | + +### Flag selection suite + +`evaluation-suites/flag-selection/` checks that an SDK sends its flag selection +on the agentless configuration request and serves only the flags the server +returns. Each case file is one provider start, so every case in a file carries +the same `providerOptions`, `expectations.configurationRequest`, and +`configurationResponse`; the validator enforces this. + +The runner cannot load the suite's `ufc-config.json` directly. It serves it +through a server double that applies the selection in the request with the +[flag selection semantics](flag-selection/README.md) and the tags in +`flag-tags.json` (a flag that is not listed has no tags), and returns only the +selected flags. For each case file: + +1. Configure the server double. When `configurationResponse.flagSelectionIgnored` + is set, it answers a valid selection with every flag and that reason in the + `DD-Flag-Selection-Ignored` header. +2. Start the provider in agentless mode against the server double, passing + `providerOptions.flagSelection` untouched to the SDK's flag selection option. + Omit the option when the field is absent. +3. Check the first configuration request: it carries each + `expectations.configurationRequest` value as a repeated `selected_flag_tags` + or `selected_flag_key_patterns` parameter, in order, or no selection + parameter for `{"flagSelection": null}`. +4. Evaluate the applicable cases as usual. A flag the server omitted evaluates + to the caller default with `ERROR` / `FLAG_NOT_FOUND`. + +An SDK sends no selection when the selection is empty after canonicalization, +breaks a limit in [flag-selection/README.md](flag-selection/README.md#semantics), +or would make the percent-encoded request URL longer than 8192 bytes +([Fastly's URL limit](https://docs.fastly.com/products/network-services-resource-limits)). +The long-URL case uses non-ASCII tags, so the URL is over that limit only once +percent-encoded. ## Origin diff --git a/ci/test_validate_fixtures.py b/ci/test_validate_fixtures.py index c5a9064f..0d084abf 100644 --- a/ci/test_validate_fixtures.py +++ b/ci/test_validate_fixtures.py @@ -168,6 +168,128 @@ def test_self_contained_suite_rejects_missing_or_extra_entries(self) -> None: validator._validate_suite(config_path, cases, self.platforms, self_contained=True) +class FlagSelectionSuiteValidationTest(unittest.TestCase): + platforms = frozenset({"php", "python"}) + path = validator.EVALUATION_SUITES_DIRECTORY / "example" / "cases" / "test-case.json" + + @staticmethod + def case(**fields: object) -> dict[str, object]: + return { + **EvaluationSuiteValidationTest.case("suite-flag"), + "providerOptions": {"flagSelection": {"tags": [" team:checkout "]}}, + "expectations": { + "configurationRequest": {"selectedFlagTags": ["team:checkout"], "selectedFlagKeyPatterns": []} + }, + **fields, + } + + def validate_case(self, case: dict[str, object]) -> None: + validator._validate_evaluation_case(self.path, 0, case, {"suite-flag": {}}, self.platforms) + + def write_suite(self, directory: str, flag_tags: object, cases: list[dict[str, object]]) -> tuple[Path, Path]: + suite = Path(directory) / "example" + (suite / "cases").mkdir(parents=True) + config_path = suite / "ufc-config.json" + config_path.write_text(json.dumps(EvaluationSuiteValidationTest.config("suite-flag")), encoding="utf-8") + (suite / "flag-tags.json").write_text(json.dumps(flag_tags), encoding="utf-8") + (suite / "cases" / "test-case.json").write_text(json.dumps(cases), encoding="utf-8") + return config_path, suite / "cases" + + def test_suite_accepts_flag_tags_for_its_own_flags(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + config_path, cases = self.write_suite( + temporary_directory, {"flags": [{"key": "suite-flag", "tags": ["team:checkout"]}]}, [self.case()] + ) + self.assertEqual( + (1, 1), validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + ) + + (config_path.parent / "unexpected.json").write_text("{}", encoding="utf-8") + with self.assertRaisesRegex(validator.FixtureValidationError, "unexpected suite entry"): + validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + + def test_flag_tags_reject_invalid_shapes_and_unknown_flags(self) -> None: + invalid = ( + ({"flags": [{"key": "root-only-flag", "tags": []}]}, "must name a flag"), + ({"flags": [{"key": "suite-flag", "tags": []}, {"key": "suite-flag", "tags": []}]}, "is repeated"), + ({"flags": [{"key": "suite-flag", "tags": ["", "team:checkout"]}]}, "non-empty strings"), + ({"flags": [{"key": "suite-flag"}]}, "only key and tags"), + ({"flags": []}, "non-empty array"), + ({"flags": [], "extra": True}, "only flags"), + ) + for flag_tags, message in invalid: + with self.subTest(flag_tags=flag_tags), tempfile.TemporaryDirectory() as temporary_directory: + config_path, cases = self.write_suite(temporary_directory, flag_tags, [self.case()]) + with self.assertRaisesRegex(validator.FixtureValidationError, message): + validator._validate_suite(config_path, cases, self.platforms, self_contained=True) + + def test_provider_start_fields_accept_valid_shapes(self) -> None: + self.validate_case(self.case()) + self.validate_case(self.case(configurationResponse={"flagSelectionIgnored": "tag_load_failed"})) + self.validate_case( + self.case( + providerOptions={"flagSelection": {"tags": [], "keyPatterns": []}}, + expectations={"configurationRequest": {"flagSelection": None}}, + ) + ) + + def test_provider_start_fields_reject_invalid_shapes(self) -> None: + no_selection = {"configurationRequest": {"flagSelection": None}} + invalid = ( + ({"providerOptions": []}, "providerOptions must be an object"), + ({"providerOptions": {"flagSelections": {}}}, "unknown fields: flagSelections"), + ({"providerOptions": {"flagSelection": {"tags": [1]}}}, "string arrays"), + ({"providerOptions": {"flagSelection": {"labels": []}}}, "string arrays"), + ({"expectations": {}}, "requires expectations.configurationRequest"), + ({"expectations": {"configurationRequest": {"flagSelection": False}}}, "flagSelection\": null"), + ( + { + "expectations": { + "configurationRequest": {"selectedFlagTags": ["b", "a"], "selectedFlagKeyPatterns": []} + } + }, + "sorted by UTF-8 bytes", + ), + ( + { + "expectations": { + "configurationRequest": {"selectedFlagTags": ["a", "a"], "selectedFlagKeyPatterns": []} + } + }, + "without duplicates", + ), + ( + {"expectations": {"configurationRequest": {"selectedFlagTags": [], "selectedFlagKeyPatterns": []}}}, + "sends no selection", + ), + ({"configurationResponse": {"flagSelectionIgnored": "unknown_reason"}}, "must be one of"), + ({"configurationResponse": {"reason": "tag_load_failed"}}, "only flagSelectionIgnored"), + ( + {"expectations": no_selection, "configurationResponse": {"flagSelectionIgnored": "tag_load_failed"}}, + "sends a selection", + ), + ) + for fields, message in invalid: + with self.subTest(fields=fields): + with self.assertRaisesRegex(validator.FixtureValidationError, message): + self.validate_case(self.case(**fields)) + + def test_cases_in_a_file_share_one_provider_start(self) -> None: + other_start = self.case( + providerOptions={"flagSelection": {"tags": ["team:search"]}}, + expectations={ + "configurationRequest": {"selectedFlagTags": ["team:search"], "selectedFlagKeyPatterns": []} + }, + ) + for cases in ([self.case(), other_start], [self.case(), EvaluationSuiteValidationTest.case("suite-flag")]): + with self.subTest(cases=cases), tempfile.TemporaryDirectory() as temporary_directory: + config_path, cases_directory = self.write_suite( + temporary_directory, {"flags": [{"key": "suite-flag", "tags": []}]}, cases + ) + with self.assertRaisesRegex(validator.FixtureValidationError, "case 1 .* must equal case 0's"): + validator._validate_suite(config_path, cases_directory, self.platforms, self_contained=True) + + class EvaluationPlatformValidationTest(unittest.TestCase): platforms = frozenset({"php", "python"}) path = validator.EVALUATION_CASES_DIRECTORY / "test-case-filter.json" diff --git a/ci/validate-fixtures.py b/ci/validate-fixtures.py index 6c290e1f..4e359084 100755 --- a/ci/validate-fixtures.py +++ b/ci/validate-fixtures.py @@ -20,6 +20,13 @@ ) VALID_VARIATION_TYPES = frozenset({"BOOLEAN", "INTEGER", "JSON", "NUMERIC", "STRING"}) FEATURE_DESTINATIONS = frozenset({"HOOK", "EXPOSURE", "EVALUATION"}) +SUITE_FLAG_TAGS_FILE = "flag-tags.json" +FLAG_SELECTION_FIELDS = frozenset({"tags", "keyPatterns"}) +CANONICAL_SELECTION_FIELDS = ("selectedFlagTags", "selectedFlagKeyPatterns") +NO_SELECTION_REQUEST = {"flagSelection": None} +FLAG_SELECTION_IGNORED_REASONS = frozenset( + {"malformed_selection", "too_many_values", "invalid_key_pattern", "invalid_tag", "tag_load_failed"} +) REQUIRED_CASE_FIELDS = frozenset( { "attributes", @@ -192,6 +199,108 @@ def _validate_evaluation_case( path, f"{location} capabilities.{capability}", result, flag, configured_flags, allow_invalid_context=True, ) + _validate_provider_start(path, location, value) + + +def _is_string_list(value: object) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _validate_provider_start(path: Path, location: str, case: dict[str, object]) -> None: + options = case.get("providerOptions", {}) + if not isinstance(options, dict): + _fail(path, f"{location} providerOptions must be an object") + unknown = sorted(set(options).difference({"flagSelection"})) + if unknown: + _fail(path, f"{location} providerOptions has unknown fields: {', '.join(unknown)}") + if "flagSelection" in options: + selection = options["flagSelection"] + if ( + not isinstance(selection, dict) + or not set(selection).issubset(FLAG_SELECTION_FIELDS) + or not all(_is_string_list(values) for values in selection.values()) + ): + _fail( + path, + f"{location} providerOptions.flagSelection must be an object with optional tags and keyPatterns " + "string arrays", + ) + + expectations = case.get("expectations", {}) + if not isinstance(expectations, dict): + _fail(path, f"{location} expectations must be an object") + sends_selection = False + if "configurationRequest" in expectations: + sends_selection = _validate_configuration_request( + path, f"{location} expectations.configurationRequest", expectations["configurationRequest"] + ) + elif "flagSelection" in options: + _fail(path, f"{location} providerOptions.flagSelection requires expectations.configurationRequest") + + if "configurationResponse" in case: + response = case["configurationResponse"] + if not isinstance(response, dict) or set(response) != {"flagSelectionIgnored"}: + _fail(path, f"{location} configurationResponse must be an object with only flagSelectionIgnored") + if response["flagSelectionIgnored"] not in FLAG_SELECTION_IGNORED_REASONS: + _fail( + path, + f"{location} configurationResponse.flagSelectionIgnored must be one of " + f"{', '.join(sorted(FLAG_SELECTION_IGNORED_REASONS))}", + ) + if not sends_selection: + _fail(path, f"{location} configurationResponse requires a configurationRequest that sends a selection") + + +def _validate_configuration_request(path: Path, location: str, request: object) -> bool: + """Validate the canonical selection on the configuration request and return whether one is sent.""" + if request == NO_SELECTION_REQUEST: + return False + if not isinstance(request, dict) or set(request) != set(CANONICAL_SELECTION_FIELDS): + _fail( + path, + f"{location} must be {{\"flagSelection\": null}} or hold only selectedFlagTags and " + "selectedFlagKeyPatterns", + ) + for field in CANONICAL_SELECTION_FIELDS: + values = request[field] + if ( + not _is_string_list(values) + or not all(values) + or values != sorted(set(values), key=lambda value: value.encode("utf-8")) + ): + _fail(path, f"{location}.{field} must hold non-empty strings sorted by UTF-8 bytes without duplicates") + if not any(request[field] for field in CANONICAL_SELECTION_FIELDS): + _fail(path, f"{location} sends no selection; use {{\"flagSelection\": null}}") + return True + + +def _provider_start(case: dict[str, object]) -> tuple[object, object, object]: + expectations = case.get("expectations", {}) + return case.get("providerOptions"), expectations.get("configurationRequest"), case.get("configurationResponse") + + +def _validate_flag_tags(path: Path, configured_flags: dict[str, object]) -> None: + if path.is_symlink() or not path.is_file(): + _fail(path, "must be a regular file, not a symlink") + value = _load_json(path) + if not isinstance(value, dict) or set(value) != {"flags"}: + _fail(path, "top-level value must be an object with only flags") + flags = value["flags"] + if not isinstance(flags, list) or not flags: + _fail(path, "flags must be a non-empty array") + keys: set[str] = set() + for index, entry in enumerate(flags): + location = f"flags[{index}]" + if not isinstance(entry, dict) or set(entry) != {"key", "tags"}: + _fail(path, f"{location} must be an object with only key and tags") + key = entry["key"] + if not isinstance(key, str) or key not in configured_flags: + _fail(path, f"{location}.key must name a flag in the suite's ufc-config.json") + if key in keys: + _fail(path, f"{location}.key {key!r} is repeated") + keys.add(key) + if not _is_string_list(entry["tags"]) or not all(entry["tags"]): + _fail(path, f"{location}.tags must be an array of non-empty strings") def _validate_result( @@ -255,12 +364,15 @@ def _validate_suite( if self_contained: suite_directory = config_path.parent - allowed_entries = {config_path.name, cases_directory.name} + allowed_entries = {config_path.name, cases_directory.name, SUITE_FLAG_TAGS_FILE} for entry in suite_directory.iterdir(): if entry.name not in allowed_entries: - _fail(entry, "unexpected suite entry; only ufc-config.json and cases/ are allowed") + _fail(entry, "unexpected suite entry; only ufc-config.json, flag-tags.json, and cases/ are allowed") configured_flags = _validate_config(config_path, _load_json(config_path)) + flag_tags_path = config_path.parent / SUITE_FLAG_TAGS_FILE + if self_contained and (flag_tags_path.exists() or flag_tags_path.is_symlink()): + _validate_flag_tags(flag_tags_path, configured_flags) evaluation_paths: list[Path] = [] for entry in sorted(cases_directory.iterdir()): if entry.is_symlink() or not entry.is_file(): @@ -281,6 +393,13 @@ def _validate_suite( for index, case in enumerate(cases): _validate_evaluation_case(path, index, case, configured_flags, platforms) + # A case file is one provider start, so its cases share the start's inputs and request. + if _provider_start(case) != _provider_start(cases[0]): + _fail( + path, + f"case {index} providerOptions, expectations.configurationRequest, and configurationResponse " + "must equal case 0's", + ) case_count += 1 return case_count, len(evaluation_paths) diff --git a/ci/validate-flag-selection.py b/ci/validate-flag-selection.py index 024cc03b..fb87c210 100644 --- a/ci/validate-flag-selection.py +++ b/ci/validate-flag-selection.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Validate the static shape of the flag selection vectors and the shared UFC tags.""" +"""Validate the static shape of the flag selection vectors.""" from __future__ import annotations @@ -10,18 +10,16 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] VECTORS_PATH = REPOSITORY_ROOT / "flag-selection" / "selection-vectors.json" -UFC_TAGS_PATH = REPOSITORY_ROOT / "flag-selection" / "ufc-config-tags.json" -UFC_CONFIG_PATH = REPOSITORY_ROOT / "ufc-config.json" QUERY_PARAMETERS = frozenset({"selected_flag_tags", "selected_flag_key_patterns"}) IGNORED_REASONS = frozenset({"malformed_selection", "too_many_values", "invalid_key_pattern", "invalid_tag"}) class VectorValidationError(ValueError): - """A flag selection file is valid JSON but violates the documented format.""" + """The vectors file is valid JSON but violates the documented format.""" -def _fail(message: str, path: Path = VECTORS_PATH) -> NoReturn: - raise VectorValidationError(f"{path.relative_to(REPOSITORY_ROOT)}: {message}") +def _fail(message: str) -> NoReturn: + raise VectorValidationError(f"{VECTORS_PATH.relative_to(REPOSITORY_ROOT)}: {message}") def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: @@ -33,62 +31,47 @@ def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object] return result -def _load(path: Path = VECTORS_PATH) -> object: +def _load() -> object: try: - return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=_reject_duplicate_keys) + return json.loads(VECTORS_PATH.read_text(encoding="utf-8"), object_pairs_hook=_reject_duplicate_keys) except json.JSONDecodeError as error: - _fail(f"invalid JSON at line {error.lineno}, column {error.colno}: {error.msg}", path) + _fail(f"invalid JSON at line {error.lineno}, column {error.colno}: {error.msg}") except VectorValidationError as error: - _fail(str(error), path) + _fail(str(error)) def _is_string_list(value: object) -> bool: return isinstance(value, list) and all(isinstance(item, str) for item in value) -def _check_keys( - location: str, value: dict[str, object], required: set[str], optional: set[str], path: Path = VECTORS_PATH -) -> None: +def _check_keys(location: str, value: dict[str, object], required: set[str], optional: set[str]) -> None: missing = sorted(required.difference(value)) if missing: - _fail(f"{location} is missing {', '.join(missing)}", path) + _fail(f"{location} is missing {', '.join(missing)}") unknown = sorted(set(value).difference(required, optional)) if unknown: - _fail(f"{location} has unknown fields {', '.join(unknown)}", path) + _fail(f"{location} has unknown fields {', '.join(unknown)}") -def _validate_flags(value: object, path: Path = VECTORS_PATH) -> set[str]: +def _validate_flags(value: object) -> set[str]: if not isinstance(value, list) or not value: - _fail("flags must be a non-empty array", path) + _fail("flags must be a non-empty array") keys: set[str] = set() for index, flag in enumerate(value): location = f"flags[{index}]" if not isinstance(flag, dict): - _fail(f"{location} must be an object", path) - _check_keys(location, flag, {"key", "tags"}, set(), path) + _fail(f"{location} must be an object") + _check_keys(location, flag, {"key", "tags"}, set()) if not isinstance(flag["key"], str): - _fail(f"{location}.key must be a string", path) + _fail(f"{location}.key must be a string") if flag["key"] in keys: - _fail(f"{location}.key {flag['key']!r} is repeated", path) + _fail(f"{location}.key {flag['key']!r} is repeated") if not _is_string_list(flag["tags"]): - _fail(f"{location}.tags must be an array of strings", path) + _fail(f"{location}.tags must be an array of strings") keys.add(flag["key"]) return keys -def _validate_ufc_tags() -> int: - tags = _load(UFC_TAGS_PATH) - if not isinstance(tags, dict): - _fail("top-level value must be an object", UFC_TAGS_PATH) - _check_keys("top-level object", tags, {"flags"}, set(), UFC_TAGS_PATH) - tagged_keys = _validate_flags(tags["flags"], UFC_TAGS_PATH) - config = json.loads(UFC_CONFIG_PATH.read_text(encoding="utf-8")) - unknown = sorted(tagged_keys.difference(config["flags"])) - if unknown: - _fail(f"flags names keys missing from ufc-config.json: {', '.join(unknown)}", UFC_TAGS_PATH) - return len(tagged_keys) - - def _validate_case(index: int, case: object, flag_keys: set[str]) -> str: location = f"cases[{index}]" if not isinstance(case, dict): @@ -182,7 +165,6 @@ def main() -> None: names.add(name) print(f"Validated {len(cases)} flag selection vectors against {len(flag_keys)} flags.") - print(f"Validated tags for {_validate_ufc_tags()} ufc-config.json flags.") if __name__ == "__main__": diff --git a/evaluation-suites/flag-selection/cases/canonical-form.json b/evaluation-suites/flag-selection/cases/canonical-form.json new file mode 100644 index 00000000..d28fb87d --- /dev/null +++ b/evaluation-suites/flag-selection/cases/canonical-form.json @@ -0,0 +1,148 @@ +[ + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by tag team:search.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by key pattern ops-*.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [" team:search ", "team:checkout", "team:search", ""], + "keyPatterns": ["\tops-*", "ops-*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout", "team:search"], + "selectedFlagKeyPatterns": ["ops-*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/empty-selection.json b/evaluation-suites/flag-selection/cases/empty-selection.json new file mode 100644 index 00000000..ecbef12e --- /dev/null +++ b/evaluation-suites/flag-selection/cases/empty-selection.json @@ -0,0 +1,140 @@ +[ + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The selection is empty, so the SDK sends none and every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": [], + "keyPatterns": [] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/interior-whitespace-tag.json b/evaluation-suites/flag-selection/cases/interior-whitespace-tag.json new file mode 100644 index 00000000..4abf3f66 --- /dev/null +++ b/evaluation-suites/flag-selection/cases/interior-whitespace-tag.json @@ -0,0 +1,145 @@ +[ + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by tag release:summer sale.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["release:summer sale"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["release:summer sale"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/key-patterns.json b/evaluation-suites/flag-selection/cases/key-patterns.json new file mode 100644 index 00000000..deeffdfe --- /dev/null +++ b/evaluation-suites/flag-selection/cases/key-patterns.json @@ -0,0 +1,144 @@ +[ + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by key pattern search-*.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by key pattern *-sale.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "keyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": [], + "selectedFlagKeyPatterns": ["*-sale", "search-*", "untagged"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/sdk-drops-long-url.json b/evaluation-suites/flag-selection/cases/sdk-drops-long-url.json new file mode 100644 index 00000000..e80b90c8 --- /dev/null +++ b/evaluation-suites/flag-selection/cases/sdk-drops-long-url.json @@ -0,0 +1,46 @@ +[ + { + "description": "Percent-encoded, the selection takes the request URL over 8192 bytes, so the SDK sends none and every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "long-01:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-02:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-03:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-04:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-05:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-06:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-07:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-08:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-09:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-10:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-11:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-12:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-13:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-14:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-15:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-16:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-17:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-18:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-19:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-20:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-21:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-22:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-23:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-24:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-25:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-26:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-27:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-28:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-29:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-30:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Percent-encoded, the selection takes the request URL over 8192 bytes, so the SDK sends none and every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "long-01:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-02:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-03:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-04:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-05:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-06:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-07:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-08:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-09:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-10:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-11:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-12:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-13:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-14:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-15:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-16:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-17:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-18:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-19:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-20:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-21:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-22:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-23:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-24:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-25:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-26:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-27:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-28:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-29:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", "long-30:ééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/sdk-drops-too-many-values.json b/evaluation-suites/flag-selection/cases/sdk-drops-too-many-values.json new file mode 100644 index 00000000..90399c3f --- /dev/null +++ b/evaluation-suites/flag-selection/cases/sdk-drops-too-many-values.json @@ -0,0 +1,134 @@ +[ + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "32 tags exceed the 31-value limit, so the SDK sends no selection and every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout", "tag-01", "tag-02", "tag-03", "tag-04", "tag-05", "tag-06", "tag-07", "tag-08", "tag-09", "tag-10", "tag-11", "tag-12", "tag-13", "tag-14", "tag-15", "tag-16", "tag-17", "tag-18", "tag-19", "tag-20", "tag-21", "tag-22", "tag-23", "tag-24", "tag-25", "tag-26", "tag-27", "tag-28", "tag-29", "tag-30", "tag-31"] + } + }, + "expectations": { + "configurationRequest": { + "flagSelection": null + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/server-ignores-selection.json b/evaluation-suites/flag-selection/cases/server-ignores-selection.json new file mode 100644 index 00000000..e800ffda --- /dev/null +++ b/evaluation-suites/flag-selection/cases/server-ignores-selection.json @@ -0,0 +1,158 @@ +[ + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "The server ignores the selection with tag_load_failed, so every flag is served.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "configurationResponse": { + "flagSelectionIgnored": "tag_load_failed" + }, + "result": { + "value": true, + "reason": "STATIC" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/tags-or-key-patterns.json b/evaluation-suites/flag-selection/cases/tags-or-key-patterns.json new file mode 100644 index 00000000..1af0831f --- /dev/null +++ b/evaluation-suites/flag-selection/cases/tags-or-key-patterns.json @@ -0,0 +1,150 @@ +[ + { + "description": "Selected by key pattern checkout-b*.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Selected by tag team:ops.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:ops"], + "keyPatterns": ["checkout-b*"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:ops"], + "selectedFlagKeyPatterns": ["checkout-b*"] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/cases/tags.json b/evaluation-suites/flag-selection/cases/tags.json new file mode 100644 index 00000000..f0d7f81b --- /dev/null +++ b/evaluation-suites/flag-selection/cases/tags.json @@ -0,0 +1,144 @@ +[ + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-banner", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Selected by tag team:checkout.", + "flag": "checkout-pricing", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": true, + "reason": "STATIC" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "search-ranking", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "summer-sale", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "ops-kill-switch", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + }, + { + "description": "Not selected: it has no selected tag and matches no key pattern, so the server omits it.", + "flag": "untagged-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "alice", + "attributes": {}, + "providerOptions": { + "flagSelection": { + "tags": ["team:checkout"] + } + }, + "expectations": { + "configurationRequest": { + "selectedFlagTags": ["team:checkout"], + "selectedFlagKeyPatterns": [] + } + }, + "result": { + "value": false, + "reason": "ERROR", + "errorCode": "FLAG_NOT_FOUND" + } + } +] diff --git a/evaluation-suites/flag-selection/flag-tags.json b/evaluation-suites/flag-selection/flag-tags.json new file mode 100644 index 00000000..fb041769 --- /dev/null +++ b/evaluation-suites/flag-selection/flag-tags.json @@ -0,0 +1,10 @@ +{ + "flags": [ + {"key": "checkout-banner", "tags": ["team:checkout"]}, + {"key": "checkout-pricing", "tags": ["team:checkout"]}, + {"key": "search-ranking", "tags": ["team:search"]}, + {"key": "summer-sale", "tags": ["release:summer sale"]}, + {"key": "ops-kill-switch", "tags": ["team:ops"]}, + {"key": "untagged-flag", "tags": []} + ] +} diff --git a/evaluation-suites/flag-selection/ufc-config.json b/evaluation-suites/flag-selection/ufc-config.json new file mode 100644 index 00000000..22c5b154 --- /dev/null +++ b/evaluation-suites/flag-selection/ufc-config.json @@ -0,0 +1,177 @@ +{ + "createdAt": "2026-11-03T00:00:00.000Z", + "format": "SERVER", + "environment": { + "name": "Test" + }, + "flags": { + "checkout-banner": { + "key": "checkout-banner", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "checkout-pricing": { + "key": "checkout-pricing", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "search-ranking": { + "key": "search-ranking", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "summer-sale": { + "key": "summer-sale", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "ops-kill-switch": { + "key": "ops-kill-switch", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + }, + "untagged-flag": { + "key": "untagged-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "everyone", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": false + } + ] + } + } +} diff --git a/flag-selection/README.md b/flag-selection/README.md index 73c8e6c9..39237bcf 100644 --- a/flag-selection/README.md +++ b/flag-selection/README.md @@ -160,23 +160,12 @@ omitted list. Then: When both are empty, the SDK sends no selection parameters. - `sdk.dropped`: the SDK drops the selection with this reason and sends none. -## Tags for `ufc-config.json` +## End-to-end suite -UFC responses carry no tags, so a mock configuration endpoint that serves the -shared [`ufc-config.json`](../ufc-config.json) cannot apply a tag selection on -its own. `ufc-config-tags.json` lists the tags of some of its flags, in the same -shape as `flags` above. A flag that is not listed has no tags. - -```json -{ - "flags": [{ "key": "basic-flag", "tags": ["team:checkout", "release:summer sale"] }] -} -``` - -A mock that serves `ufc-config.json` builds the flag set from every key in -`ufc-config.json` and the tags listed here, applies the selection with the -semantics above, and returns only the selected entries of `flags`. The vectors -do not run against this flag set. +[`evaluation-suites/flag-selection/`](../evaluation-suites/flag-selection/) +runs selections through an SDK against a server double that applies these +semantics to that suite's `ufc-config.json` and `flag-tags.json`. See +[flag selection suite](../README.md#flag-selection-suite). ## Consumers @@ -196,5 +185,4 @@ distinct, and present in `flags`. For `sdk`, it checks that every `query` case has one and no `rawQuery` case does, that `sdk.selection` lists hold non-empty strings sorted by UTF-8 bytes without duplicates, that `sdk.selection` appears only on `applied` cases, and that `sdk.dropped` equals `expected.reason`. It -does not recompute expected outcomes. For `ufc-config-tags.json`, it checks the -same `flags` shape and that every key names a flag in `ufc-config.json`. +does not recompute expected outcomes. diff --git a/flag-selection/ufc-config-tags.json b/flag-selection/ufc-config-tags.json deleted file mode 100644 index a88b7453..00000000 --- a/flag-selection/ufc-config-tags.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "flags": [ - {"key": "basic-flag", "tags": ["team:checkout", "release:summer sale"]}, - {"key": "numeric_flag", "tags": ["team:checkout"]}, - {"key": "experiment-flag", "tags": ["team:search"]} - ] -}