diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 277408f8d82..641b874b47c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,9 @@ /tests/ @DataDog/system-tests-reviewers # FFE +/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/.github/workflows/update-ffe-fixtures.yml @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core +/utils/scripts/update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/docker_fixtures/_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/parametric/Endpoints/FfeTestApi.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core @@ -23,6 +26,7 @@ /utils/build/docker/php/common/ffe.php @DataDog/feature-flagging-and-experimentation-sdk @DataDog/system-tests-core /tests/ffe/ @DataDog/feature-flagging-and-experimentation-sdk /tests/test_the_test/test_mock_ffe_agentless_backend.py @DataDog/feature-flagging-and-experimentation-sdk +/tests/test_the_test/test_update_ffe_fixtures.py @DataDog/feature-flagging-and-experimentation-sdk /tests/parametric/test_ffe/ @DataDog/feature-flagging-and-experimentation-sdk # SDK cap diff --git a/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml new file mode 100644 index 00000000000..7aeba37ded2 --- /dev/null +++ b/.github/chainguard/self.update-ffe-fixtures.create-pr.sts.yaml @@ -0,0 +1,14 @@ +--- +issuer: https://token.actions.githubusercontent.com + +subject: repo:DataDog/system-tests:ref:refs/heads/main + +claim_pattern: + event_name: (workflow_dispatch|schedule) + ref: refs/heads/main + ref_protected: "true" + job_workflow_ref: DataDog/system-tests/\.github/workflows/update-ffe-fixtures\.yml@refs/heads/main + +permissions: + contents: write + pull_requests: write diff --git a/.github/workflows/update-ffe-fixtures.yml b/.github/workflows/update-ffe-fixtures.yml new file mode 100644 index 00000000000..778db406a4b --- /dev/null +++ b/.github/workflows/update-ffe-fixtures.yml @@ -0,0 +1,87 @@ +--- +name: Update FFE fixtures + +on: + workflow_dispatch: + inputs: + fixture_ref: + description: Branch, tag, or commit from DataDog/ffe-system-test-data + required: true + default: main + type: string + schedule: + - cron: "0 0 * * 0" + +concurrency: + group: update-ffe-fixtures + cancel-in-progress: false + +jobs: + update-ffe-fixtures: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + + - name: Update fixture snapshot + id: fixtures + env: + FFE_FIXTURE_REF: ${{ inputs.fixture_ref || 'main' }} + run: python3 utils/scripts/update_ffe_fixtures.py --ref "$FFE_FIXTURE_REF" + + - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 + if: steps.fixtures.outputs.changed == 'true' + id: octo-sts + with: + scope: DataDog/system-tests + policy: self.update-ffe-fixtures.create-pr + + - name: Check for an open fixture update PR + if: steps.fixtures.outputs.changed == 'true' + id: existing-pr + env: + GH_TOKEN: ${{ steps.octo-sts.outputs.token }} + run: | + count=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --head actions/update-ffe-fixtures --state open \ + --json headRefName,headRepositoryOwner,isCrossRepository \ + --jq '[.[] | select( + .headRefName == "actions/update-ffe-fixtures" and + .headRepositoryOwner.login == "DataDog" and + (.isCrossRepository | not) + )] | length') + echo "count=$count" >> "$GITHUB_OUTPUT" + + - name: Create Pull Request + if: >- + steps.fixtures.outputs.changed == 'true' && steps.existing-pr.outputs.count == '0' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.octo-sts.outputs.token }} + branch: actions/update-ffe-fixtures + base: main + commit-message: "test(ffe): update canonical fixtures" + title: "test(ffe): update canonical fixtures" + draft: true + sign-commits: true + add-paths: tests/parametric/test_ffe/ffe-system-test-data + body: | + ## Motivation + + Keep SDK behavior consistent with the shared FFE evaluator contract. + + ## Changes + + Refresh the checked-in snapshot to [${{ steps.fixtures.outputs.source_commit }}](https://github.com/DataDog/ffe-system-test-data/commit/${{ steps.fixtures.outputs.source_commit }}), containing ${{ steps.fixtures.outputs.fixture_count }} evaluation cases. + + Ordinary test runs use the snapshot without network access. Resolve newly exposed evaluator failures before merging; correct shared expectations upstream only when they are wrong. Scheduled updates leave an existing update PR untouched to preserve review and fixes. + + ## Workflow + + 1. Keep this PR as a draft while validating the updated fixtures against the supported SDKs. + 2. Follow the repository's [review workflow](https://github.com/DataDog/system-tests/blob/main/.github/pull_request_template.md). diff --git a/tests/ffe/README.md b/tests/ffe/README.md index be43490db3c..f1205b765d5 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -12,6 +12,35 @@ This directory contains system tests for the Feature Flags & Experimentation (FF | `test_flag_eval_metrics.py` | Evaluation metrics (OTel counter) | | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds | +## Canonical parametric fixtures + +The parametric evaluation suite reads the checked-in snapshot under +`tests/parametric/test_ffe/ffe-system-test-data`. Its upstream source and exact +commit are recorded in `SOURCE.md` inside that directory. + +Change shared evaluator cases in +[`DataDog/ffe-system-test-data`](https://github.com/DataDog/ffe-system-test-data) +first, then refresh the snapshot: + +```bash +python3 utils/scripts/update_ffe_fixtures.py --ref +``` + +The updater imports `ufc-config.json` and `evaluation-cases/*.json` only. Ordinary +test runs use those checked-in files without submodules or network access. A +weekly/manual workflow opens a signed draft PR when their contents change and +leaves any existing fixture update PR untouched. Review and resolve newly exposed +SDK failures before merging an update. + +Dynamic evaluation checks each case's value, reason, and explicit error code. +Cases with optional `capabilities` still run their baseline `result`; enabling +capability-specific provider options requires separate harness support. +Remote Configuration tests and the mocked agentless backend read the same UFC. +Serial-ID metadata and span-enrichment tests select their flag definitions from +the shared corpus using their dedicated evaluation-case files. Their integration +harnesses retain trace construction and metadata assertions; no separate local +configuration copies are needed. + ## Running FFE Tests ```bash @@ -197,7 +226,7 @@ The tests use `interfaces.agent.get_metrics()` to retrieve metrics from the agen ### 1. Test returns `parse_error` instead of expected reason - Check UFC fixture format (especially `totalShards` placement) -- Verify the fixture matches `flags-v1.json` format in dd-trace-py +- Verify the fixture matches the canonical `ffe-system-test-data/ufc-config.json` format ### 2. Test returns `static` instead of `split` - Need multiple variations with different shard ranges diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index bcf93e46a17..5e53cb1b8a8 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -242,7 +242,7 @@ def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None: class FlagevaluationEgressContract: """One flag-evaluation contract inherited by each supported topology adapter.""" - # Agentless scenarios preload flags-v1.json before the weblog starts. Use the + # Agentless scenarios preload the canonical UFC before the weblog starts. Use the # same known fixture key in every topology; the Agent scenario installs an # equivalent fixture through Remote Config below. flag_key = "empty-targeting-key-flag" diff --git a/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md new file mode 100644 index 00000000000..0a907783d5f --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/SOURCE.md @@ -0,0 +1,12 @@ +# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: cb4a8620adb90f4ada757e1042e346efa8035364 + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. diff --git a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json similarity index 74% rename from tests/parametric/test_ffe/test-case-boolean-false-assignment.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json index cdd2b0a513d..ab718f5bee1 100644 --- a/tests/parametric/test_ffe/test-case-boolean-false-assignment.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-false-assignment.json @@ -1,38 +1,41 @@ [ { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "alice", "attributes": { "should_disable_feature": true }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": false - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "bob", "attributes": { "should_disable_feature": false }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "boolean-false-assignment", - "variationType": "BOOLEAN", - "defaultValue": true, - "targetingKey": "charlie", "attributes": { "unknown_attribute": "value" }, + "defaultValue": true, + "flag": "boolean-false-assignment", "result": { + "reason": "DEFAULT", "value": true - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json similarity index 72% rename from tests/parametric/test_ffe/test-case-boolean-one-of-matches.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json index f616614d936..392ec97ec0f 100644 --- a/tests/parametric/test_ffe/test-case-boolean-one-of-matches.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-boolean-one-of-matches.json @@ -1,192 +1,208 @@ [ { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "one_of_flag": "True" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "not_matches_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "not_matches_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "haley", "attributes": { "not_matches_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 4 - } + }, + "targetingKey": "haley", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "ivy", "attributes": { "not_one_of_flag": true }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "ivy", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "julia", "attributes": { "not_one_of_flag": false }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "julia", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "kim", "attributes": { "not_one_of_flag": "False" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "kim", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "lucas", "attributes": { "not_one_of_flag": "true" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "lucas", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "mike", "attributes": { "not_one_of_flag": "false" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "mike", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "nicole", "attributes": { "null_flag": "null" }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "TARGETING_MATCH", "value": 5 - } + }, + "targetingKey": "nicole", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "owen", "attributes": { "null_flag": null }, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "owen", + "variationType": "INTEGER" }, { - "flag": "boolean-one-of-matches", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "pete", "attributes": {}, + "defaultValue": 0, + "flag": "boolean-one-of-matches", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "pete", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-comparator-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json index 2d94f30eb30..e4daa2de7c2 100644 --- a/tests/parametric/test_ffe/test-case-comparator-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-comparator-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "small" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "medium" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "size": 25 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "large" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "comparator-operator-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "unknown", + "flag": "comparator-operator-test", "result": { + "reason": "DEFAULT", "value": "unknown" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-dependent-flags.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-dependent-flags.json new file mode 100644 index 00000000000..b91d49d41f6 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-dependent-flags.json @@ -0,0 +1,112 @@ +[ + { + "description": "A root flag matches a direct prerequisite by variant key.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-direct-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "direct-match" + }, + "targetingKey": "dependent-direct", + "variationType": "STRING" + }, + { + "description": "ONE_OF does not match when the prerequisite variant key is absent from the allowed list.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-one-of-miss-root", + "result": { + "reason": "STATIC", + "value": "one-of-fallback" + }, + "targetingKey": "dependent-one-of-miss", + "variationType": "STRING" + }, + { + "description": "NOT_ONE_OF does not match when the prerequisite variant key is present in the excluded list.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-not-one-of-hit-root", + "result": { + "reason": "STATIC", + "value": "not-one-of-fallback" + }, + "targetingKey": "dependent-not-one-of-hit", + "variationType": "STRING" + }, + { + "description": "A depth-two tree evaluates its leaf and intermediate prerequisite before matching the root.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-transitive-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "transitive-match" + }, + "targetingKey": "dependent-transitive", + "variationType": "STRING" + }, + { + "description": "A false attribute condition short-circuits before an otherwise fatal missing prerequisite and selects the root fallback.", + "attributes": { + "gate": "closed" + }, + "defaultValue": "caller-default", + "flag": "dependent-short-circuit-root", + "result": { + "reason": "STATIC", + "value": "short-circuit-fallback" + }, + "targetingKey": "dependent-short-circuit", + "variationType": "STRING" + }, + { + "description": "Two membership conditions against the same prerequisite evaluate successfully.", + "attributes": {}, + "defaultValue": "caller-default", + "flag": "dependent-memoized-root", + "result": { + "reason": "TARGETING_MATCH", + "value": "memoized-match" + }, + "targetingKey": "dependent-memoized", + "variationType": "STRING" + }, + { + "description": "A successful prerequisite followed by a missing prerequisite aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "atomic-default", + "flag": "dependent-atomic-error-root", + "result": { + "reason": "ERROR", + "value": "atomic-default" + }, + "targetingKey": "dependent-atomic-error", + "variationType": "STRING" + }, + { + "description": "With maxDependencyDepth 2, a third dependency edge aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "depth-default", + "flag": "dependent-depth-three-root", + "result": { + "reason": "ERROR", + "value": "depth-default" + }, + "targetingKey": "dependent-over-depth", + "variationType": "STRING" + }, + { + "description": "A dependency cycle aborts the graph and returns the caller default.", + "attributes": {}, + "defaultValue": "cycle-default", + "flag": "dependent-cycle-a", + "result": { + "reason": "ERROR", + "value": "cycle-default" + }, + "targetingKey": "dependent-cycle", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-disabled-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json similarity index 59% rename from tests/parametric/test_ffe/test-case-disabled-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json index 0da79189ade..208e5c96943 100644 --- a/tests/parametric/test_ffe/test-case-disabled-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-disabled-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "disabled_flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "disabled_flag", "result": { + "reason": "DISABLED", "value": 0 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json similarity index 63% rename from tests/parametric/test_ffe/test-case-empty-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json index 52100b1fe47..18ee5381fdc 100644 --- a/tests/parametric/test_ffe/test-case-empty-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "empty_flag", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default_value", + "flag": "empty_flag", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-empty-string-variation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json similarity index 75% rename from tests/parametric/test_ffe/test-case-empty-string-variation.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json index 1af431f3db4..f1fa3994af2 100644 --- a/tests/parametric/test_ffe/test-case-empty-string-variation.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-empty-string-variation.json @@ -1,38 +1,41 @@ [ { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "empty_user", "attributes": { "content_type": "minimal" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "empty_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "full_user", "attributes": { "content_type": "full" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "TARGETING_MATCH", "value": "detailed_content" - } + }, + "targetingKey": "full_user", + "variationType": "STRING" }, { - "flag": "empty-string-variation", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "default_user", "attributes": { "content_type": "unknown" }, + "defaultValue": "default_value", + "flag": "empty-string-variation", "result": { + "reason": "DEFAULT", "value": "default_value" - } + }, + "targetingKey": "default_user", + "variationType": "STRING" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json new file mode 100644 index 00000000000..8120c5fb73d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-experimentation-opt-out.json @@ -0,0 +1,210 @@ +[ + { + "flag": "unknown-opt-out-flag", "variationType": "STRING", "defaultValue": "default", "targetingKey": "subject", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "default", "reason": "ERROR", "errorCode": "FLAG_NOT_FOUND" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": null, + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": {}, "reason": "ERROR", "errorCode": "TARGETING_KEY_MISSING" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "red", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "19", + "attributes": { "country": "UK", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "red", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "DEFAULT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "green", "reason": "TARGETING_MATCH" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": { "a": "kümmert", "b": "schön" }, "reason": "STATIC" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": {}, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 2, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": {}, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 1, "reason": "SPLIT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "1", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 2, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "numeric_flag", "variationType": "NUMERIC", "defaultValue": 0, "targetingKey": "alice", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": 3.1415926, "reason": "STATIC" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "kill-switch", "variationType": "BOOLEAN", "defaultValue": false, "targetingKey": "alice", + "attributes": { "country": "US", "dd_experimentation_opt_out_at": "2026-10-19T12:00:00Z" }, + "result": { "value": true, "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": false, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T12:00:00.123456789Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2026-10-19T08:00:00-04:00" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "2099-01-01T00:00:00Z" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "DEFAULT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": "" }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": true }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": null }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": 1792411200 }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": false }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "new-user-onboarding", "variationType": "STRING", "defaultValue": "default", "targetingKey": "alice", + "attributes": { "email": "alice@mycompany.com", "dd_experimentation_opt_out_at": null }, + "result": { "value": "green", "reason": "TARGETING_MATCH" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": "default", "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "special-characters", "variationType": "JSON", "defaultValue": {}, "targetingKey": "ben", + "attributes": { "dd_experimentation_opt_out_at": "not-a-timestamp" }, + "result": { "value": { "a": "піклуватися", "b": "любов" }, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": {}, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": { "timestamp": "2026-10-19T12:00:00Z" } }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + }, + { + "flag": "integer-flag", "variationType": "INTEGER", "defaultValue": 0, "targetingKey": "5", + "attributes": { "dd_experimentation_opt_out_at": ["2026-10-19T12:00:00Z"] }, + "result": { "value": 1, "reason": "SPLIT" }, + "capabilities": { + "experimentationOptOutImmediate": { "value": 0, "reason": "ERROR", "errorCode": "INVALID_CONTEXT" } + } + } +] diff --git a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json similarity index 73% rename from tests/parametric/test_ffe/test-case-falsy-value-assignments.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json index 4ad259620d8..6a8cee51bcc 100644 --- a/tests/parametric/test_ffe/test-case-falsy-value-assignments.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-falsy-value-assignments.json @@ -1,38 +1,41 @@ [ { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "zero_user", "attributes": { "plan_tier": "free" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 0 - } + }, + "targetingKey": "zero_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "premium_user", "attributes": { "plan_tier": "premium" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "TARGETING_MATCH", "value": 100 - } + }, + "targetingKey": "premium_user", + "variationType": "INTEGER" }, { - "flag": "falsy-value-assignments", - "variationType": "INTEGER", - "defaultValue": 999, - "targetingKey": "unknown_user", "attributes": { "plan_tier": "trial" }, + "defaultValue": 999, + "flag": "falsy-value-assignments", "result": { + "reason": "DEFAULT", "value": 999 - } + }, + "targetingKey": "unknown_user", + "variationType": "INTEGER" } -] \ No newline at end of file +] diff --git a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json similarity index 72% rename from tests/parametric/test_ffe/test-case-flag-with-empty-string.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json index 0bf562efa35..6a5d9cd62fd 100644 --- a/tests/parametric/test_ffe/test-case-flag-with-empty-string.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-flag-with-empty-string.json @@ -1,24 +1,26 @@ [ { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "TARGETING_MATCH", "value": "" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "empty_string_flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": {}, + "defaultValue": "default", + "flag": "empty_string_flag", "result": { + "reason": "SPLIT", "value": "non_empty" - } + }, + "targetingKey": "bob", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-integer-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json similarity index 66% rename from tests/parametric/test_ffe/test-case-integer-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json index 5e930885b44..8ff11aec1fc 100644 --- a/tests/parametric/test_ffe/test-case-integer-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-integer-flag.json @@ -1,244 +1,267 @@ [ { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "debra", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "1", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "1", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "2", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "2", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "3", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "3", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "4", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "4", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "5", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "5", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "6", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "6", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "7", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "7", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "8", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "8", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "9", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "9", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "10", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "10", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "11", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "11", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "12", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "12", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "13", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "13", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "14", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "14", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "15", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "15", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "16", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 2 - } + }, + "targetingKey": "16", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "17", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "17", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "18", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "18", + "variationType": "INTEGER" }, { - "flag": "integer-flag", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "19", "attributes": {}, + "defaultValue": 0, + "flag": "integer-flag", "result": { + "reason": "SPLIT", "value": 1 - } + }, + "targetingKey": "19", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json new file mode 100644 index 00000000000..be226381455 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-condition-operands.json @@ -0,0 +1,59 @@ +[ + { + "description": "GT requires a numeric configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "age": 42 + }, + "defaultValue": "default", + "flag": "invalid-gt-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-gt", + "variationType": "STRING" + }, + { + "description": "ONE_OF requires an array configured operand. A flag with a scalar operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "role": "admin" + }, + "defaultValue": "default", + "flag": "invalid-one-of-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-one-of", + "variationType": "STRING" + }, + { + "description": "IS_NULL requires a boolean configured operand. A flag with a string operand must return a parse error rather than falling through to the catch-all allocation.", + "attributes": { + "deleted_at": null + }, + "defaultValue": "default", + "flag": "invalid-is-null-condition-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "invalid-is-null", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same configuration must still evaluate after malformed condition operands are isolated.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json new file mode 100644 index 00000000000..3ad596cbf06 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-regex-isolation.json @@ -0,0 +1,17 @@ +[ + { + "description": "The malformed flag uses the invalid regex *@datadoghq.com before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": { + "email": "harish@datadoghq.com" + }, + "defaultValue": "default", + "flag": "invalid-regex-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "harish", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json new file mode 100644 index 00000000000..ca8553f099c --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-invalid-shard-bounds-isolation.json @@ -0,0 +1,40 @@ +[ + { + "description": "The malformed flag has totalShards above the unsigned 32-bit range. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "overflow-total-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "The malformed flag has a negative shard range start. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "negative-shard-range-start-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after invalid shard-bound flags are skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-kill-switch-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-kill-switch-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json index 8f34a1bc3af..3af700dc711 100644 --- a/tests/parametric/test_ffe/test-case-kill-switch-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-kill-switch-flag.json @@ -1,290 +1,314 @@ [ { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "alice", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "bob", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "barbara", "attributes": { - "email": "barbara@example.com", - "country": "canada" + "country": "canada", + "email": "barbara@example.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "barbara", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "charlie", "attributes": { "age": 40 }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "charlie", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "debra", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "1", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "1", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "2", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 50 + "age": 50, + "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "3", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "4", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "5", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "6", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "7", "attributes": { - "country": "US", - "age": 12 + "age": 12, + "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "7", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "8", "attributes": { - "country": "Italy", - "age": 60 + "age": 60, + "country": "Italy" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "8", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "9", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "10", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "10", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "11", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "11", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "12", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "TARGETING_MATCH", "value": true - } + }, + "targetingKey": "13", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "14", "attributes": {}, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "14", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "15", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "16", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "17", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "18", + "variationType": "BOOLEAN" }, { - "flag": "kill-switch", - "variationType": "BOOLEAN", - "defaultValue": false, - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": false, + "flag": "kill-switch", "result": { + "reason": "STATIC", "value": false - } + }, + "targetingKey": "19", + "variationType": "BOOLEAN" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json new file mode 100644 index 00000000000..41672357c23 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-malformed-flag-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has allocations encoded as a string instead of an array. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "malformed-allocations-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the malformed flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json new file mode 100644 index 00000000000..3dba337ad02 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-microsecond-date-flag.json @@ -0,0 +1,39 @@ +[ + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "country": "US" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "1.0.0" + }, + "defaultValue": "unknown", + "flag": "microsecond-date-test", + "result": { + "reason": "DEFAULT", + "value": "active" + }, + "targetingKey": "charlie", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json new file mode 100644 index 00000000000..d6dfeb180ea --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-missing-split-shards-isolation.json @@ -0,0 +1,27 @@ +[ + { + "description": "The malformed flag has a split that omits required shards. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "missing-split-shards-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the missing-shards flag is skipped.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json similarity index 69% rename from tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json index 8ed3e2a024f..bb08523de4e 100644 --- a/tests/parametric/test_ffe/test-case-new-user-onboarding-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-new-user-onboarding-flag.json @@ -1,318 +1,344 @@ [ { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "green" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "debra", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "debra", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "purple" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "zach", "attributes": { - "id": "override-id", - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com", + "id": "override-id" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "Zach", "attributes": { - "email": "test@test.com", + "age": 25, "country": "Mexico", - "age": 25 + "email": "test@test.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "Zach", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "1", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "1", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "2", "attributes": { "country": "Mexico" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "2", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "3", "attributes": { - "country": "UK", - "age": 33 + "age": 33, + "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "3", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "4", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "4", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "5", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "5", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "6", "attributes": { "country": "Germany" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "6", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "7", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "7", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "8", "attributes": { "country": "Italy" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "8", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "9", "attributes": { "email": "email@email.com" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "9", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "10", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "10", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "11", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "11", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "12", "attributes": { "country": "US" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "12", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "13", "attributes": { "country": "Canada" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "blue" - } + }, + "targetingKey": "13", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "14", "attributes": {}, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "14", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "15", "attributes": { "country": "Denmark" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "yellow" - } + }, + "targetingKey": "15", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "16", "attributes": { "country": "Norway" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "16", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "17", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "control" - } + }, + "targetingKey": "17", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "18", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "DEFAULT", "value": "default" - } + }, + "targetingKey": "18", + "variationType": "STRING" }, { - "flag": "new-user-onboarding", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "19", "attributes": { "country": "UK" }, + "defaultValue": "default", + "flag": "new-user-onboarding", "result": { + "reason": "TARGETING_MATCH", "value": "red" - } + }, + "targetingKey": "19", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/test-case-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json similarity index 68% rename from tests/parametric/test_ffe/test-case-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json index 132c39db32a..7d985b8efd4 100644 --- a/tests/parametric/test_ffe/test-case-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "hello": "world" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "hello": "world" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "hello": "world" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "hello": "world" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-case-null-operator-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json similarity index 67% rename from tests/parametric/test_ffe/test-case-null-operator-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json index 09e5d78dacd..8063ff8fee3 100644 --- a/tests/parametric/test_ffe/test-case-null-operator-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-operator-flag.json @@ -1,64 +1,69 @@ [ { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "alice", "attributes": { - "size": 5, - "country": "US" + "country": "US", + "size": 5 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "bob", "attributes": { - "size": 10, - "country": "Canada" + "country": "Canada", + "size": 10 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "charlie", "attributes": { "size": null }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "david", "attributes": { "size": 26 }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "new" - } + }, + "targetingKey": "david", + "variationType": "STRING" }, { - "flag": "null-operator-test", - "variationType": "STRING", - "defaultValue": "default-null", - "targetingKey": "elize", "attributes": { "country": "UK" }, + "defaultValue": "default-null", + "flag": "null-operator-test", "result": { + "reason": "TARGETING_MATCH", "value": "old" - } + }, + "targetingKey": "elize", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json new file mode 100644 index 00000000000..15f4af5a77d --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-shard-range-isolation.json @@ -0,0 +1,15 @@ +[ + { + "description": "The malformed flag has a null shard range before a catch-all allocation. SDKs must return a parse error without poisoning the full configuration.", + "attributes": {}, + "defaultValue": "default", + "flag": "null-shard-range-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json new file mode 100644 index 00000000000..a2226d1b21e --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-null-targeting-key.json @@ -0,0 +1,40 @@ +[ + { + "attributes": {}, + "defaultValue": 42, + "flag": "numeric_flag", + "result": { + "reason": "STATIC", + "value": 3.1415926 + }, + "targetingKey": null, + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "France" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "ERROR", + "value": "default" + }, + "targetingKey": null, + "variationType": "STRING" + }, + { + "attributes": { + "country": "France", + "email": "null@mycompany.com" + }, + "defaultValue": "default", + "flag": "new-user-onboarding", + "result": { + "reason": "TARGETING_MATCH", + "value": "green" + }, + "targetingKey": null, + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json similarity index 52% rename from tests/parametric/test_ffe/test-case-numeric-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json index 757f0f70e55..cc52dcdff4d 100644 --- a/tests/parametric/test_ffe/test-case-numeric-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", "attributes": { - "email": "alice@mycompany.com", - "country": "US" + "country": "US", + "email": "alice@mycompany.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "alice", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", "attributes": { - "email": "bob@example.com", - "country": "Canada" + "country": "Canada", + "email": "bob@example.com" }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "bob", + "variationType": "NUMERIC" }, { - "flag": "numeric_flag", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": 0, + "flag": "numeric_flag", "result": { + "reason": "STATIC", "value": 3.1415926 - } + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json new file mode 100644 index 00000000000..d8a7e8ba092 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of-default.json @@ -0,0 +1,13 @@ +[ + { + "attributes": {}, + "defaultValue": 0, + "flag": "numeric-one-of", + "result": { + "reason": "DEFAULT", + "value": 0 + }, + "targetingKey": "race-default", + "variationType": "INTEGER" + } +] diff --git a/tests/parametric/test_ffe/test-case-numeric-one-of.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json similarity index 70% rename from tests/parametric/test_ffe/test-case-numeric-one-of.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json index 9eaccbc477c..bfa957335d3 100644 --- a/tests/parametric/test_ffe/test-case-numeric-one-of.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-numeric-one-of.json @@ -1,86 +1,93 @@ [ { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "alice", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "alice", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "bob", "attributes": { "number": 2 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "DEFAULT", "value": 0 - } + }, + "targetingKey": "bob", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "charlie", "attributes": { "number": 3 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "charlie", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "derek", "attributes": { "number": 4 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 3 - } + }, + "targetingKey": "derek", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "erica", "attributes": { "number": "1" }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "erica", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "frank", "attributes": { "number": 1 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 1 - } + }, + "targetingKey": "frank", + "variationType": "INTEGER" }, { - "flag": "numeric-one-of", - "variationType": "INTEGER", - "defaultValue": 0, - "targetingKey": "george", "attributes": { "number": 123456789 }, + "defaultValue": 0, + "flag": "numeric-one-of", "result": { + "reason": "TARGETING_MATCH", "value": 2 - } + }, + "targetingKey": "george", + "variationType": "INTEGER" } ] diff --git a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json similarity index 73% rename from tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json index 03f03ab1ecb..e99a207b6da 100644 --- a/tests/parametric/test_ffe/test-case-of-7-empty-targeting-key.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-of-7-empty-targeting-key.json @@ -1,12 +1,13 @@ [ { - "flag": "empty-targeting-key-flag", - "variationType": "STRING", - "defaultValue": "default", - "targetingKey": "", "attributes": {}, + "defaultValue": "default", + "flag": "empty-targeting-key-flag", "result": { + "reason": "STATIC", "value": "on-value" - } + }, + "targetingKey": "", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json new file mode 100644 index 00000000000..b01f6f1e5ca --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-regex-flag.json @@ -0,0 +1,122 @@ +[ + { + "attributes": { + "comma_regex_case": "xxaaay" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "comma-quantifier" + }, + "targetingKey": "comma-quantifier", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "prefix exact suffix" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "literal-whitespace" + }, + "targetingKey": "literal-whitespace", + "variationType": "STRING" + }, + { + "attributes": { + "whitespace_regex_case": "exact" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "trimmed-whitespace-does-not-match", + "variationType": "STRING" + }, + { + "attributes": { + "email": "user.name+tag@capture.example" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "capturing-groups" + }, + "targetingKey": "capture", + "variationType": "STRING" + }, + { + "attributes": { + "email": "llega mañana temprano" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "unicode-word-boundary" + }, + "targetingKey": "unicode", + "variationType": "STRING" + }, + { + "attributes": { + "email": "alice@example.com", + "version": "1.15.0" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "partial-example" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "email": "bob@test.com", + "version": "0.20.1" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "test" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "charlie", + "variationType": "STRING" + }, + { + "attributes": { + "email": "derek@gmail.com", + "version": "2.1.13" + }, + "defaultValue": "none", + "flag": "regex-flag", + "result": { + "reason": "DEFAULT", + "value": "none" + }, + "targetingKey": "derek", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json new file mode 100644 index 00000000000..bc6fe8b92bc --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-comparison-flag.json @@ -0,0 +1,169 @@ +[ + { + "attributes": { + "app_version": "1.2.3-alpha.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "1.2.3-alpha.5" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "legacy" + }, + "targetingKey": "bob", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "carol", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "2.9.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "dave", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "eve", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "3.4.5-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "frank", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.0" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta" + }, + "targetingKey": "grace", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "henry", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6-rc.1+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "launch" + }, + "targetingKey": "mia", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "4.5.6" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "future" + }, + "targetingKey": "iris", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "5.6.7" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "jane", + "variationType": "STRING" + }, + { + "attributes": { + "app_version": "not-a-semver" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "kyle", + "variationType": "STRING" + }, + { + "attributes": {}, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "liam", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json new file mode 100644 index 00000000000..4f00961d9f9 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-categories.json @@ -0,0 +1,120 @@ +[ + { + "description": "A non-parseable SemVer comparand (\"not-a-version\") rejects the flag at parse time with a PARSE_ERROR rather than matching or coercing.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-syntax-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-syntax", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with repeated hyphens inside its prerelease identifiers (\"1.2.3----R-S.12.9.1--.12+meta\") is valid and matches normally.", + "attributes": { + "app_version": "1.2.3----R-S.12.9.1--.12+meta" + }, + "defaultValue": "unknown", + "flag": "semver-valid-comparand-hyphenated-prerelease-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "matched" + }, + "targetingKey": "valid-hyphenated-prerelease", + "variationType": "STRING" + }, + { + "description": "A SemVer comparand with consecutive dots in its prerelease identifiers (\"1.0.0-alpha..1\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-consecutive-dots-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-consecutive-dots", + "variationType": "STRING" + }, + { + "description": "A Ruby-style dot-suffixed SemVer comparand (\"1.2.3.DEV\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-dot-suffix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-dot-suffix", + "variationType": "STRING" + }, + { + "description": "A v-prefixed SemVer comparand (\"v1.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-vprefix-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-vprefix", + "variationType": "STRING" + }, + { + "description": "A leading-zero core component in the SemVer comparand (\"01.2.3\") rejects the flag at parse time with a PARSE_ERROR.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-leading-zero-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-leading-zero", + "variationType": "STRING" + }, + { + "description": "A non-string (JSON number) SemVer comparand rejects the flag at parse time with a PARSE_ERROR rather than coercing the number to a string.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-non-string-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-non-string", + "variationType": "STRING" + }, + { + "description": "The rejected invalid-comparand flags must not poison the rest of the configuration: a neighboring valid SemVer flag still evaluates normally.", + "attributes": { + "app_version": "2.3.4" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "stable" + }, + "targetingKey": "after-invalid-categories", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json new file mode 100644 index 00000000000..cbddbb87d03 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-invalid-comparand-waterfall.json @@ -0,0 +1,17 @@ +[ + { + "description": "An invalid configured SemVer comparand in a non-last allocation must reject the whole flag at parse time, even though a valid default allocation follows it in the waterfall. An implementation that skips the invalid allocation and falls through to the default would return \"fallback\" with a STATIC reason instead of an error.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-waterfall-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "waterfall", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json new file mode 100644 index 00000000000..ef027a06660 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-precedence-flag.json @@ -0,0 +1,170 @@ +[ + { + "description": "Multi-digit numeric prerelease identifiers compare by numeric value, not lexicographically: 1.0.0-beta.2 must precede 1.0.0-beta.11. An implementation that string-compares identifiers would treat \"2\" as greater than \"11\" and fail to match.", + "attributes": { + "app_version": "10.0.0-beta.2" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "numeric-prerelease" + }, + "targetingKey": "numeric-prerelease", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for equality: 1.0.0+linux and 1.0.0+darwin have equal precedence, so SEMVER_EQ matches.", + "attributes": { + "app_version": "20.0.0+linux" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-equal-precedence" + }, + "targetingKey": "build-equal-precedence", + "variationType": "STRING" + }, + { + "description": "Dotted/multi-segment build metadata is ignored for equality: 4.0.0+exp.sha.5114f85 equals 4.0.0.", + "attributes": { + "app_version": "30.0.0+exp.sha.5114f85" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-dotted-eq" + }, + "targetingKey": "build-dotted-eq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LTE: 4.0.0+build.42 <= 4.0.0 is true (match).", + "attributes": { + "app_version": "40.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-lte" + }, + "targetingKey": "build-lte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GTE: 4.0.0+build.42 >= 4.0.0 is true (match).", + "attributes": { + "app_version": "50.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "build-gte" + }, + "targetingKey": "build-gte", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_NEQ: 4.0.0+build.42 != 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "60.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-neq", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_LT: 4.0.0+build.42 < 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "70.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-lt", + "variationType": "STRING" + }, + { + "description": "Build metadata is ignored for SEMVER_GT: 4.0.0+build.42 > 4.0.0 is false (no match), so the evaluation falls through to the default.", + "attributes": { + "app_version": "80.0.0+build.42" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "build-gt", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: 1.0.0-alpha precedes 1.0.0-alpha.1 (fewer identifiers first), so SEMVER_LT matches.", + "attributes": { + "app_version": "90.0.0-alpha" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-before-alpha-1" + }, + "targetingKey": "alpha-before-alpha-1", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: numeric identifiers have lower precedence than alphanumeric (1.0.0-alpha.1 precedes 1.0.0-alpha.beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "91.0.0-alpha.1" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-1-before-alpha-beta" + }, + "targetingKey": "alpha-1-before-alpha-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: identifiers are compared lexicographically (1.0.0-alpha.beta precedes 1.0.0-beta), so SEMVER_LT matches.", + "attributes": { + "app_version": "92.0.0-alpha.beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "alpha-beta-before-beta" + }, + "targetingKey": "alpha-beta-before-beta", + "variationType": "STRING" + }, + { + "description": "Alphanumeric prerelease ordering: a prerelease version precedes the corresponding release (1.0.0-beta precedes 1.0.0), so SEMVER_LT matches.", + "attributes": { + "app_version": "93.0.0-beta" + }, + "defaultValue": "unknown", + "flag": "semver-precedence-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "beta-before-release" + }, + "targetingKey": "beta-before-release", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json new file mode 100644 index 00000000000..9b4bfea3710 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-validation-flag.json @@ -0,0 +1,255 @@ +[ + { + "description": "The maximum safe integer value is valid in every core version component.", + "attributes": { + "app_version": "9007199254740991.9007199254740991.9007199254740991" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "maximum" + }, + "targetingKey": "max-core", + "variationType": "STRING" + }, + { + "description": "Semver does not accept a v prefix.", + "attributes": { + "app_version": "v1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "v-prefix", + "variationType": "STRING" + }, + { + "description": "Core numeric components cannot have leading zeros.", + "attributes": { + "app_version": "01.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "core-leading-zero", + "variationType": "STRING" + }, + { + "description": "Numeric prerelease identifiers cannot have leading zeros.", + "attributes": { + "app_version": "1.2.3-01" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "prerelease-leading-zero", + "variationType": "STRING" + }, + { + "description": "Prerelease identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3-alpha..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease-identifier", + "variationType": "STRING" + }, + { + "description": "Build metadata identifiers cannot be empty.", + "attributes": { + "app_version": "1.2.3+build..1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build-identifier", + "variationType": "STRING" + }, + { + "description": "Prerelease and build identifiers are ASCII-only.", + "attributes": { + "app_version": "1.2.3-α" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-ascii-prerelease", + "variationType": "STRING" + }, + { + "description": "An empty string is not a valid SemVer version.", + "attributes": { + "app_version": "" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-string", + "variationType": "STRING" + }, + { + "description": "A leading zero in the minor core component is rejected.", + "attributes": { + "app_version": "1.02.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "minor-leading-zero", + "variationType": "STRING" + }, + { + "description": "A leading zero in the patch core component is rejected.", + "attributes": { + "app_version": "1.2.03" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "patch-leading-zero", + "variationType": "STRING" + }, + { + "description": "A trailing prerelease delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3-" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-prerelease", + "variationType": "STRING" + }, + { + "description": "A trailing build delimiter with no identifiers is rejected.", + "attributes": { + "app_version": "1.2.3+" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "empty-build", + "variationType": "STRING" + }, + { + "description": "Underscores are not permitted in prerelease identifiers.", + "attributes": { + "app_version": "1.2.3-alpha_1" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "underscore-prerelease", + "variationType": "STRING" + }, + { + "description": "More than one build metadata delimiter is rejected.", + "attributes": { + "app_version": "1.2.3-alpha+build+other" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "multiple-build-delimiters", + "variationType": "STRING" + }, + { + "description": "Leading whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": " 1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "leading-whitespace", + "variationType": "STRING" + }, + { + "description": "Trailing whitespace is not accepted by the SemVer parser.", + "attributes": { + "app_version": "1.2.3 " + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "trailing-whitespace", + "variationType": "STRING" + }, + { + "description": "A non-string attribute (JSON number) does not match any SemVer condition and falls through to the default rather than erroring.", + "attributes": { + "app_version": 1.2 + }, + "defaultValue": "unknown", + "flag": "semver-comparison-test", + "result": { + "reason": "DEFAULT", + "value": "unknown" + }, + "targetingKey": "non-string-attribute", + "variationType": "STRING" + }, + { + "description": "A flag with an invalid configured SemVer comparand returns a parse error while the rest of the configuration remains usable.", + "attributes": { + "app_version": "1.2.3" + }, + "defaultValue": "unknown", + "flag": "semver-invalid-comparand-test", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "unknown" + }, + "targetingKey": "invalid-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json new file mode 100644 index 00000000000..bac7fdade7b --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-semver-version-parts.json @@ -0,0 +1,118 @@ +[ + { + "description": "A one-part version is normalized with missing minor and patch parts: \"18\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "one-part-input", + "app_version": "18" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-input" + }, + "targetingKey": "one-part-input", + "variationType": "STRING" + }, + { + "description": "A one-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-one-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "one-part-comparand" + }, + "targetingKey": "one-part-comparand", + "variationType": "STRING" + }, + { + "description": "A two-part version is normalized with a missing patch part: \"18.0\" compares equal to \"18.0.0\".", + "attributes": { + "scenario": "two-part-input", + "app_version": "18.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-input" + }, + "targetingKey": "two-part-input", + "variationType": "STRING" + }, + { + "description": "A two-part configured comparand is valid and normalized: \"18.0.0\" compares equal to configured \"18.0\".", + "attributes": { + "app_version": "18.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-two-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "two-part-comparand" + }, + "targetingKey": "two-part-comparand", + "variationType": "STRING" + }, + { + "description": "A four-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "four-part-input", + "app_version": "18.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-input" + }, + "targetingKey": "four-part-input", + "variationType": "STRING" + }, + { + "description": "A four-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-four-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "four-part-comparand" + }, + "targetingKey": "four-part-comparand", + "variationType": "STRING" + }, + { + "description": "A five-part version is accepted as an attribute and compares above a canonical three-part lower version.", + "attributes": { + "scenario": "five-part-input", + "app_version": "18.0.0.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-input-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-input" + }, + "targetingKey": "five-part-input", + "variationType": "STRING" + }, + { + "description": "A five-part configured comparand is accepted and compares below a canonical three-part higher version.", + "attributes": { + "app_version": "19.0.0" + }, + "defaultValue": "unknown", + "flag": "semver-version-parts-five-part-comparand-test", + "result": { + "reason": "TARGETING_MATCH", + "value": "five-part-comparand" + }, + "targetingKey": "five-part-comparand", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json new file mode 100644 index 00000000000..8bfadabb0e7 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-serial-id-metadata.json @@ -0,0 +1,97 @@ +[ + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-zero-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 0, + "doLog": true + } + }, + { + "flag": "serial-id-missing-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + "result": { + "value": "control", + "reason": "STATIC", + "serialId": null, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-vip", + "attributes": { + "segment": "vip" + }, + "result": { + "value": "vip", + "reason": "TARGETING_MATCH", + "serialId": 201, + "doLog": true + } + }, + { + "flag": "serial-id-targeting-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-standard", + "attributes": { + "segment": "standard" + }, + "result": { + "value": "standard", + "reason": "STATIC", + "serialId": 200, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + }, + { + "flag": "serial-id-basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-repeat", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 42, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json new file mode 100644 index 00000000000..dd988ba8583 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-span-enrichment-inputs.json @@ -0,0 +1,67 @@ +[ + { + "flag": "basic-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 100, + "doLog": true + } + }, + { + "flag": "experiment-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": "treatment", + "reason": "STATIC", + "serialId": 108, + "doLog": true + } + }, + { + "flag": "no-log-flag", + "variationType": "BOOLEAN", + "defaultValue": false, + "targetingKey": "user-no-log", + "attributes": {}, + "result": { + "value": true, + "reason": "STATIC", + "serialId": 200, + "doLog": false + } + }, + { + "flag": "multi-serial-flag-1", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 1, + "reason": "STATIC", + "serialId": 128, + "doLog": true + } + }, + { + "flag": "multi-serial-flag-2", + "variationType": "INTEGER", + "defaultValue": 0, + "targetingKey": "user-max-test", + "attributes": {}, + "result": { + "value": 2, + "reason": "STATIC", + "serialId": 130, + "doLog": true + } + } +] diff --git a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json similarity index 65% rename from tests/parametric/test_ffe/test-case-start-and-end-date-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json index 7a48ec35886..53bf8859cdd 100644 --- a/tests/parametric/test_ffe/test-case-start-and-end-date-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-start-and-end-date-flag.json @@ -1,40 +1,43 @@ [ { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "alice", "attributes": { - "version": "1.15.0", - "country": "US" + "country": "US", + "version": "1.15.0" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "alice", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "bob", "attributes": { - "version": "0.20.1", - "country": "Canada" + "country": "Canada", + "version": "0.20.1" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "bob", + "variationType": "STRING" }, { - "flag": "start-and-end-date-test", - "variationType": "STRING", - "defaultValue": "unknown", - "targetingKey": "charlie", "attributes": { "version": "2.1.13" }, + "defaultValue": "unknown", + "flag": "start-and-end-date-test", "result": { + "reason": "DEFAULT", "value": "current" - } + }, + "targetingKey": "charlie", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json new file mode 100644 index 00000000000..1bc2c914534 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-fields-tolerance.json @@ -0,0 +1,16 @@ +[ + { + "description": "The flag and surrounding UFC config include unknown object fields that SDKs must ignore while evaluating the known schema fields.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "unknown-fields-tolerance-flag", + "result": { + "reason": "TARGETING_MATCH", + "value": "on" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json new file mode 100644 index 00000000000..c37fde4d573 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-unknown-operator-isolation.json @@ -0,0 +1,31 @@ +[ + { + "description": "The first allocation uses an unknown operator and the second is a catch-all trap. SDKs must return a parse error without falling through to the catch-all allocation.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "operator-grease-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": "default" + }, + "targetingKey": "alice", + "variationType": "STRING" + }, + { + "description": "A valid flag in the same config must still evaluate after the unknown-operator flag is skipped.", + "attributes": { + "country": "US" + }, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "alice", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json new file mode 100644 index 00000000000..ca3b775ad61 --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-case-variant-type-mismatch.json @@ -0,0 +1,27 @@ +[ + { + "description": "A flag whose variation violates its declared type returns a parse error while the rest of the configuration remains usable.", + "attributes": {}, + "defaultValue": 0, + "flag": "variant-type-mismatch-flag", + "result": { + "errorCode": "PARSE_ERROR", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "user-1", + "variationType": "INTEGER" + }, + { + "description": "A valid flag in the same configuration continues to evaluate when a malformed neighboring flag returns a parse error.", + "attributes": {}, + "defaultValue": "default", + "flag": "valid-flag-after-invalid-config", + "result": { + "reason": "STATIC", + "value": "expected" + }, + "targetingKey": "valid-neighbor", + "variationType": "STRING" + } +] diff --git a/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json new file mode 100644 index 00000000000..0be74f1f7ac --- /dev/null +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-flag-that-does-not-exist.json @@ -0,0 +1,46 @@ +[ + { + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "alice", + "variationType": "NUMERIC" + }, + { + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "bob", + "variationType": "NUMERIC" + }, + { + "attributes": { + "age": 50 + }, + "defaultValue": 0, + "flag": "flag-that-does-not-exist", + "result": { + "errorCode": "FLAG_NOT_FOUND", + "reason": "ERROR", + "value": 0 + }, + "targetingKey": "charlie", + "variationType": "NUMERIC" + } +] diff --git a/tests/parametric/test_ffe/test-json-config-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json similarity index 62% rename from tests/parametric/test_ffe/test-json-config-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json index ecc799546b0..45a9c2ee973 100644 --- a/tests/parametric/test_ffe/test-json-config-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-json-config-flag.json @@ -1,72 +1,76 @@ [ { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 1.0, "integer": 1, - "string": "one", - "float": 1.0 + "string": "one" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "foo": "bar" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "SPLIT", "value": { + "float": 2.0, "integer": 2, - "string": "two", - "float": 2.0 + "string": "two" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" }, { - "flag": "json-config-flag", - "variationType": "JSON", - "defaultValue": { - "foo": "bar" - }, - "targetingKey": "diana", "attributes": { "Force Empty": true }, + "defaultValue": { + "foo": "bar" + }, + "flag": "json-config-flag", "result": { + "reason": "TARGETING_MATCH", "value": {} - } + }, + "targetingKey": "diana", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-no-allocations-flag.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json similarity index 70% rename from tests/parametric/test_ffe/test-no-allocations-flag.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json index 45867e5897c..2b03b9a39a1 100644 --- a/tests/parametric/test_ffe/test-no-allocations-flag.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-no-allocations-flag.json @@ -1,52 +1,55 @@ [ { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "US", + "email": "alice@mycompany.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "alice", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", + "attributes": { + "country": "Canada", + "email": "bob@example.com" + }, "defaultValue": { "message": "Hello, world!" }, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "bob", + "variationType": "JSON" }, { - "flag": "no_allocations_flag", - "variationType": "JSON", - "defaultValue": { - "message": "Hello, world!" - }, - "targetingKey": "charlie", "attributes": { "age": 50 }, + "defaultValue": { + "message": "Hello, world!" + }, + "flag": "no_allocations_flag", "result": { + "reason": "DEFAULT", "value": { "message": "Hello, world!" } - } + }, + "targetingKey": "charlie", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json index 120647ec3b5..1ed7795bb95 100644 --- a/tests/parametric/test_ffe/test-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-special-characters.json @@ -1,54 +1,58 @@ [ { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ash", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "kümmert", "b": "schön" } - } + }, + "targetingKey": "ash", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "ben", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "піклуватися", "b": "любов" } - } + }, + "targetingKey": "ben", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "cameron", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "照顾", "b": "漂亮" } - } + }, + "targetingKey": "cameron", + "variationType": "JSON" }, { - "flag": "special-characters", - "variationType": "JSON", - "defaultValue": {}, - "targetingKey": "darryl", "attributes": {}, + "defaultValue": {}, + "flag": "special-characters", "result": { + "reason": "SPLIT", "value": { "a": "🤗", "b": "🌸" } - } + }, + "targetingKey": "darryl", + "variationType": "JSON" } ] diff --git a/tests/parametric/test_ffe/test-string-with-special-characters.json b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json similarity index 77% rename from tests/parametric/test_ffe/test-string-with-special-characters.json rename to tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json index e56322d44f0..32397a48398 100644 --- a/tests/parametric/test_ffe/test-string-with-special-characters.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/evaluation-cases/test-string-with-special-characters.json @@ -1,794 +1,860 @@ [ { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_spaces", "attributes": { "string_with_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " a b c d e f " - } + }, + "targetingKey": "string_with_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_space", "attributes": { "string_with_only_one_space": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_one_space", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_spaces", "attributes": { "string_with_only_multiple_spaces": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": " " - } + }, + "targetingKey": "string_with_only_multiple_spaces", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dots", "attributes": { "string_with_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ".a.b.c.d.e.f." - } + }, + "targetingKey": "string_with_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dot", "attributes": { "string_with_only_one_dot": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "." - } + }, + "targetingKey": "string_with_only_one_dot", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dots", "attributes": { "string_with_only_multiple_dots": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "......." - } + }, + "targetingKey": "string_with_only_multiple_dots", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_comas", "attributes": { "string_with_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",a,b,c,d,e,f," - } + }, + "targetingKey": "string_with_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_coma", "attributes": { "string_with_only_one_coma": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "," - } + }, + "targetingKey": "string_with_only_one_coma", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_comas", "attributes": { "string_with_only_multiple_comas": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ",,,,,,," - } + }, + "targetingKey": "string_with_only_multiple_comas", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_colons", "attributes": { "string_with_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":a:b:c:d:e:f:" - } + }, + "targetingKey": "string_with_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_colon", "attributes": { "string_with_only_one_colon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":" - } + }, + "targetingKey": "string_with_only_one_colon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_colons", "attributes": { "string_with_only_multiple_colons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ":::::::" - } + }, + "targetingKey": "string_with_only_multiple_colons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_semicolons", "attributes": { "string_with_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";a;b;c;d;e;f;" - } + }, + "targetingKey": "string_with_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_semicolon", "attributes": { "string_with_only_one_semicolon": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";" - } + }, + "targetingKey": "string_with_only_one_semicolon", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_semicolons", "attributes": { "string_with_only_multiple_semicolons": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ";;;;;;;" - } + }, + "targetingKey": "string_with_only_multiple_semicolons", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_slashes", "attributes": { "string_with_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/a/b/c/d/e/f/" - } + }, + "targetingKey": "string_with_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_slash", "attributes": { "string_with_only_one_slash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "/" - } + }, + "targetingKey": "string_with_only_one_slash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_slashes", "attributes": { "string_with_only_multiple_slashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "///////" - } + }, + "targetingKey": "string_with_only_multiple_slashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dashes", "attributes": { "string_with_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-a-b-c-d-e-f-" - } + }, + "targetingKey": "string_with_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dash", "attributes": { "string_with_only_one_dash": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-" - } + }, + "targetingKey": "string_with_only_one_dash", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dashes", "attributes": { "string_with_only_multiple_dashes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "-------" - } + }, + "targetingKey": "string_with_only_multiple_dashes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_underscores", "attributes": { "string_with_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_a_b_c_d_e_f_" - } + }, + "targetingKey": "string_with_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_underscore", "attributes": { "string_with_only_one_underscore": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_" - } + }, + "targetingKey": "string_with_only_one_underscore", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_underscores", "attributes": { "string_with_only_multiple_underscores": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "_______" - } + }, + "targetingKey": "string_with_only_multiple_underscores", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_plus_signs", "attributes": { "string_with_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+a+b+c+d+e+f+" - } + }, + "targetingKey": "string_with_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_plus_sign", "attributes": { "string_with_only_one_plus_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+" - } + }, + "targetingKey": "string_with_only_one_plus_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_plus_signs", "attributes": { "string_with_only_multiple_plus_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "+++++++" - } + }, + "targetingKey": "string_with_only_multiple_plus_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_equal_signs", "attributes": { "string_with_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=a=b=c=d=e=f=" - } + }, + "targetingKey": "string_with_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_equal_sign", "attributes": { "string_with_only_one_equal_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=" - } + }, + "targetingKey": "string_with_only_one_equal_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_equal_signs", "attributes": { "string_with_only_multiple_equal_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "=======" - } + }, + "targetingKey": "string_with_only_multiple_equal_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_dollar_signs", "attributes": { "string_with_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$a$b$c$d$e$f$" - } + }, + "targetingKey": "string_with_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_dollar_sign", "attributes": { "string_with_only_one_dollar_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$" - } + }, + "targetingKey": "string_with_only_one_dollar_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_dollar_signs", "attributes": { "string_with_only_multiple_dollar_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "$$$$$$$" - } + }, + "targetingKey": "string_with_only_multiple_dollar_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_at_signs", "attributes": { "string_with_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@a@b@c@d@e@f@" - } + }, + "targetingKey": "string_with_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_at_sign", "attributes": { "string_with_only_one_at_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@" - } + }, + "targetingKey": "string_with_only_one_at_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_at_signs", "attributes": { "string_with_only_multiple_at_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "@@@@@@@" - } + }, + "targetingKey": "string_with_only_multiple_at_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_amp_signs", "attributes": { "string_with_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&a&b&c&d&e&f&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026a\u0026b\u0026c\u0026d\u0026e\u0026f\u0026" + }, + "targetingKey": "string_with_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_amp_sign", "attributes": { "string_with_only_one_amp_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026" + }, + "targetingKey": "string_with_only_one_amp_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_amp_signs", "attributes": { "string_with_only_multiple_amp_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { - "value": "&&&&&&&" - } + "reason": "TARGETING_MATCH", + "value": "\u0026\u0026\u0026\u0026\u0026\u0026\u0026" + }, + "targetingKey": "string_with_only_multiple_amp_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_hash_signs", "attributes": { "string_with_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#a#b#c#d#e#f#" - } + }, + "targetingKey": "string_with_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_hash_sign", "attributes": { "string_with_only_one_hash_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#" - } + }, + "targetingKey": "string_with_only_one_hash_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_hash_signs", "attributes": { "string_with_only_multiple_hash_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "#######" - } + }, + "targetingKey": "string_with_only_multiple_hash_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_percentage_signs", "attributes": { "string_with_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%a%b%c%d%e%f%" - } + }, + "targetingKey": "string_with_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_percentage_sign", "attributes": { "string_with_only_one_percentage_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%" - } + }, + "targetingKey": "string_with_only_one_percentage_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_percentage_signs", "attributes": { "string_with_only_multiple_percentage_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "%%%%%%%" - } + }, + "targetingKey": "string_with_only_multiple_percentage_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_tilde_signs", "attributes": { "string_with_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~a~b~c~d~e~f~" - } + }, + "targetingKey": "string_with_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_tilde_sign", "attributes": { "string_with_only_one_tilde_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~" - } + }, + "targetingKey": "string_with_only_one_tilde_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_tilde_signs", "attributes": { "string_with_only_multiple_tilde_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "~~~~~~~" - } + }, + "targetingKey": "string_with_only_multiple_tilde_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_asterix_signs", "attributes": { "string_with_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*a*b*c*d*e*f*" - } + }, + "targetingKey": "string_with_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_asterix_sign", "attributes": { "string_with_only_one_asterix_sign": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*" - } + }, + "targetingKey": "string_with_only_one_asterix_sign", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_asterix_signs", "attributes": { "string_with_only_multiple_asterix_signs": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "*******" - } + }, + "targetingKey": "string_with_only_multiple_asterix_signs", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_single_quotes", "attributes": { "string_with_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'a'b'c'd'e'f'" - } + }, + "targetingKey": "string_with_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_single_quote", "attributes": { "string_with_only_one_single_quote": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'" - } + }, + "targetingKey": "string_with_only_one_single_quote", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_single_quotes", "attributes": { "string_with_only_multiple_single_quotes": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "'''''''" - } + }, + "targetingKey": "string_with_only_multiple_single_quotes", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_question_marks", "attributes": { "string_with_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?a?b?c?d?e?f?" - } + }, + "targetingKey": "string_with_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_question_mark", "attributes": { "string_with_only_one_question_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "?" - } + }, + "targetingKey": "string_with_only_one_question_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_question_marks", "attributes": { "string_with_only_multiple_question_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "???????" - } + }, + "targetingKey": "string_with_only_multiple_question_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_exclamation_marks", "attributes": { "string_with_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!a!b!c!d!e!f!" - } + }, + "targetingKey": "string_with_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_exclamation_mark", "attributes": { "string_with_only_one_exclamation_mark": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!" - } + }, + "targetingKey": "string_with_only_one_exclamation_mark", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_exclamation_marks", "attributes": { "string_with_only_multiple_exclamation_marks": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "!!!!!!!" - } + }, + "targetingKey": "string_with_only_multiple_exclamation_marks", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_opening_parentheses", "attributes": { "string_with_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(a(b(c(d(e(f(" - } + }, + "targetingKey": "string_with_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_opening_parenthese", "attributes": { "string_with_only_one_opening_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(" - } + }, + "targetingKey": "string_with_only_one_opening_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_opening_parentheses", "attributes": { "string_with_only_multiple_opening_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": "(((((((" - } + }, + "targetingKey": "string_with_only_multiple_opening_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_closing_parentheses", "attributes": { "string_with_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")a)b)c)d)e)f)" - } + }, + "targetingKey": "string_with_closing_parentheses", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_one_closing_parenthese", "attributes": { "string_with_only_one_closing_parenthese": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")" - } + }, + "targetingKey": "string_with_only_one_closing_parenthese", + "variationType": "STRING" }, { - "flag": "string_flag_with_special_characters", - "variationType": "STRING", - "defaultValue": "default_value", - "targetingKey": "string_with_only_multiple_closing_parentheses", "attributes": { "string_with_only_multiple_closing_parentheses": true }, + "defaultValue": "default_value", + "flag": "string_flag_with_special_characters", "result": { + "reason": "TARGETING_MATCH", "value": ")))))))" - } + }, + "targetingKey": "string_with_only_multiple_closing_parentheses", + "variationType": "STRING" } ] diff --git a/tests/parametric/test_ffe/flags-v1.json b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json similarity index 54% rename from tests/parametric/test_ffe/flags-v1.json rename to tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json index f01bfe2ef86..ca100adf610 100644 --- a/tests/parametric/test_ffe/flags-v1.json +++ b/tests/parametric/test_ffe/ffe-system-test-data/ufc-config.json @@ -2,8 +2,14 @@ "createdAt": "2024-04-17T19:40:53.716Z", "format": "SERVER", "environment": { - "name": "Test" + "name": "Test", + "unknownEnvironmentField": "ignored" }, + "evaluatorParams": { + "maxDependencyDepth": 2 + }, + "unknownTopLevelField": "ignored", + "unknownTopLevelNull": null, "flags": { "empty_flag": { "key": "empty_flag", @@ -78,9 +84,109 @@ "test": { "key": "test", "value": "test" + }, + "capturing-groups": { + "key": "capturing-groups", + "value": "capturing-groups" + }, + "unicode-word-boundary": { + "key": "unicode-word-boundary", + "value": "unicode-word-boundary" + }, + "comma-quantifier": { + "key": "comma-quantifier", + "value": "comma-quantifier" + }, + "literal-whitespace": { + "key": "literal-whitespace", + "value": "literal-whitespace" } }, "allocations": [ + { + "key": "comma-quantifier", + "rules": [ + { + "conditions": [ + { + "attribute": "comma_regex_case", + "operator": "MATCHES", + "value": "a{1,3}" + } + ] + } + ], + "splits": [ + { + "variationKey": "comma-quantifier", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "literal-whitespace", + "rules": [ + { + "conditions": [ + { + "attribute": "whitespace_regex_case", + "operator": "MATCHES", + "value": " exact " + } + ] + } + ], + "splits": [ + { + "variationKey": "literal-whitespace", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "capturing-groups", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "^([[:alnum:]._%+-]+)@capture\\.example$" + } + ] + } + ], + "splits": [ + { + "variationKey": "capturing-groups", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "unicode-word-boundary", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "(?u)\\bmañana\\b" + } + ] + } + ], + "splits": [ + { + "variationKey": "unicode-word-boundary", + "shards": [] + } + ], + "doLog": true + }, { "key": "partial-example", "rules": [ @@ -612,122 +718,187 @@ } ] }, - "start-and-end-date-test": { - "key": "start-and-end-date-test", + "semver-comparison-test": { + "key": "semver-comparison-test", "enabled": true, "variationType": "STRING", "variations": { - "old": { - "key": "old", - "value": "old" + "legacy": { + "key": "legacy", + "value": "legacy" }, - "current": { - "key": "current", - "value": "current" + "stable": { + "key": "stable", + "value": "stable" }, - "new": { - "key": "new", - "value": "new" + "beta": { + "key": "beta", + "value": "beta" + }, + "launch": { + "key": "launch", + "value": "launch" + }, + "future": { + "key": "future", + "value": "future" + }, + "maximum": { + "key": "maximum", + "value": "maximum" } }, "allocations": [ { - "key": "old-versions", + "key": "maximum-version", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "9007199254740991.9007199254740991.9007199254740991" + } + ] + } + ], "splits": [ { - "variationKey": "old", + "variationKey": "maximum", "shards": [] } ], - "endAt": "2002-10-31T09:00:00.594Z", "doLog": true }, { - "key": "future-versions", + "key": "legacy-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "1.2.3-alpha.4" + } + ] + }, + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "1.2.3-alpha.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "2.3.4" + } + ] + } + ], "splits": [ { - "variationKey": "new", + "variationKey": "legacy", "shards": [] } ], - "startAt": "2052-10-31T09:00:00.594Z", "doLog": true }, { - "key": "current-versions", + "key": "stable-versions", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "2.3.4" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "3.4.5-beta.2" + } + ] + } + ], "splits": [ { - "variationKey": "current", + "variationKey": "stable", "shards": [] } ], - "startAt": "2022-10-31T09:00:00.594Z", - "endAt": "2050-10-31T09:00:00.594Z", "doLog": true - } - ] - }, - "null-operator-test": { - "key": "null-operator-test", - "enabled": true, - "variationType": "STRING", - "variations": { - "old": { - "key": "old", - "value": "old" }, - "new": { - "key": "new", - "value": "new" - } - }, - "allocations": [ { - "key": "null-operator", + "key": "beta-versions", "rules": [ { "conditions": [ { - "attribute": "size", - "operator": "IS_NULL", - "value": true + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "3.4.5-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "4.5.6-rc.1" } ] - }, + } + ], + "splits": [ + { + "variationKey": "beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "launch-version", + "rules": [ { "conditions": [ { - "attribute": "size", - "operator": "LT", - "value": 10 + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "4.5.6-rc.1" } ] } ], "splits": [ { - "variationKey": "old", + "variationKey": "launch", "shards": [] } ], "doLog": true }, { - "key": "not-null-operator", + "key": "future-versions", "rules": [ { "conditions": [ { - "attribute": "size", - "operator": "IS_NULL", - "value": false + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "4.5.6-rc.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "5.6.7" } ] } ], "splits": [ { - "variationKey": "new", + "variationKey": "future", "shards": [] } ], @@ -735,108 +906,316 @@ } ] }, - "new-user-onboarding": { - "key": "new-user-onboarding", + "semver-invalid-comparand-test": { + "key": "semver-invalid-comparand-test", "enabled": true, "variationType": "STRING", + "comment": "This flag deliberately uses an invalid configured SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", "variations": { - "control": { - "key": "control", - "value": "control" - }, - "red": { - "key": "red", - "value": "red" - }, - "blue": { - "key": "blue", - "value": "blue" - }, - "green": { - "key": "green", - "value": "green" - }, - "yellow": { - "key": "yellow", - "value": "yellow" - }, - "purple": { - "key": "purple", - "value": "purple" + "matched": { + "key": "matched", + "value": "matched" } }, "allocations": [ { - "key": "id rule", + "key": "invalid-comparand", "rules": [ { "conditions": [ { - "attribute": "id", - "operator": "MATCHES", - "value": "zach" + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" } ] } ], "splits": [ { - "variationKey": "purple", + "variationKey": "matched", "shards": [] } ], - "doLog": false + "doLog": true + } + ] + }, + "semver-invalid-comparand-waterfall-test": { + "key": "semver-invalid-comparand-waterfall-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately places an invalid configured SemVer comparand in a non-last allocation of a waterfall, followed by a valid default allocation. SDKs must reject the whole flag during parsing rather than skipping the invalid allocation and falling through to the default.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" }, + "fallback": { + "key": "fallback", + "value": "fallback" + } + }, + "allocations": [ { - "key": "internal users", + "key": "invalid-comparand", "rules": [ { "conditions": [ { - "attribute": "email", - "operator": "MATCHES", - "value": "@mycompany.com" + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18446744073709551616.0.0" } ] } ], "splits": [ { - "variationKey": "green", + "variationKey": "matched", "shards": [] } ], - "doLog": false + "doLog": true }, { - "key": "experiment", - "rules": [ + "key": "default", + "rules": [], + "splits": [ { - "conditions": [ - { - "attribute": "country", - "operator": "NOT_ONE_OF", - "value": [ - "US", - "Canada", - "Mexico" - ] - } - ] + "variationKey": "fallback", + "shards": [] } ], - "splits": [ - { - "variationKey": "control", - "shards": [ - { - "salt": "traffic-new-user-onboarding-experiment", - "totalShards": 10000, - "ranges": [ - { - "start": 0, - "end": 6000 - } + "doLog": true + } + ] + }, + "start-and-end-date-test": { + "key": "start-and-end-date-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "old": { + "key": "old", + "value": "old" + }, + "current": { + "key": "current", + "value": "current" + }, + "new": { + "key": "new", + "value": "new" + } + }, + "allocations": [ + { + "key": "old-versions", + "splits": [ + { + "variationKey": "old", + "shards": [] + } + ], + "endAt": "2002-10-31T09:00:00.594Z", + "doLog": true + }, + { + "key": "future-versions", + "splits": [ + { + "variationKey": "new", + "shards": [] + } + ], + "startAt": "2052-10-31T09:00:00.594Z", + "doLog": true + }, + { + "key": "current-versions", + "splits": [ + { + "variationKey": "current", + "shards": [] + } + ], + "startAt": "2022-10-31T09:00:00.594Z", + "endAt": "2050-10-31T09:00:00.594Z", + "doLog": true + } + ] + }, + "null-operator-test": { + "key": "null-operator-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "old": { + "key": "old", + "value": "old" + }, + "new": { + "key": "new", + "value": "new" + } + }, + "allocations": [ + { + "key": "null-operator", + "rules": [ + { + "conditions": [ + { + "attribute": "size", + "operator": "IS_NULL", + "value": true + } + ] + }, + { + "conditions": [ + { + "attribute": "size", + "operator": "LT", + "value": 10 + } + ] + } + ], + "splits": [ + { + "variationKey": "old", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "not-null-operator", + "rules": [ + { + "conditions": [ + { + "attribute": "size", + "operator": "IS_NULL", + "value": false + } + ] + } + ], + "splits": [ + { + "variationKey": "new", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "new-user-onboarding": { + "key": "new-user-onboarding", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "red": { + "key": "red", + "value": "red" + }, + "blue": { + "key": "blue", + "value": "blue" + }, + "green": { + "key": "green", + "value": "green" + }, + "yellow": { + "key": "yellow", + "value": "yellow" + }, + "purple": { + "key": "purple", + "value": "purple" + } + }, + "allocations": [ + { + "key": "id rule", + "rules": [ + { + "conditions": [ + { + "attribute": "id", + "operator": "MATCHES", + "value": "zach" + } + ] + } + ], + "splits": [ + { + "variationKey": "purple", + "shards": [] + } + ], + "doLog": false + }, + { + "key": "internal users", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "@mycompany.com" + } + ] + } + ], + "splits": [ + { + "variationKey": "green", + "shards": [] + } + ], + "doLog": false + }, + { + "key": "experiment", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": "NOT_ONE_OF", + "value": [ + "US", + "Canada", + "Mexico" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "traffic-new-user-onboarding-experiment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 6000 + } ] }, { @@ -3293,6 +3672,2327 @@ "doLog": true } ] + }, + "microsecond-date-test": { + "key": "microsecond-date-test", + "enabled": true, + "variationType": "STRING", + "variations": { + "expired": { + "key": "expired", + "value": "expired" + }, + "active": { + "key": "active", + "value": "active" + }, + "future": { + "key": "future", + "value": "future" + } + }, + "allocations": [ + { + "key": "expired-allocation", + "splits": [ + { + "variationKey": "expired", + "shards": [] + } + ], + "endAt": "2002-10-31T09:00:00.594321Z", + "doLog": true + }, + { + "key": "future-allocation", + "splits": [ + { + "variationKey": "future", + "shards": [] + } + ], + "startAt": "2052-10-31T09:00:00.123456Z", + "doLog": true + }, + { + "key": "active-allocation", + "splits": [ + { + "variationKey": "active", + "shards": [] + } + ], + "startAt": "2022-10-31T09:00:00.235982Z", + "endAt": "2050-10-31T09:00:00.987654Z", + "doLog": true + } + ] + }, + "unknown-fields-tolerance-flag": { + "key": "unknown-fields-tolerance-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately contains unknown fields at every fixed-schema UFC level. SDKs must ignore those fields and evaluate the flag normally.", + "unknownFlagField": "ignored", + "variations": { + "on": { + "key": "on", + "value": "on", + "unknownVariationField": "ignored" + } + }, + "allocations": [ + { + "key": "unknown-fields-allocation", + "unknownAllocationField": "ignored", + "rules": [ + { + "unknownRuleField": "ignored", + "conditions": [ + { + "attribute": "country", + "operator": "ONE_OF", + "value": [ + "US" + ], + "unknownConditionField": "ignored" + } + ] + } + ], + "splits": [ + { + "variationKey": "on", + "unknownSplitField": "ignored", + "shards": [ + { + "salt": "unknown-fields-tolerance-flag", + "totalShards": 10000, + "unknownShardField": "ignored", + "ranges": [ + { + "start": 0, + "end": 10000, + "unknownRangeField": "ignored" + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "operator-grease-flag": { + "key": "operator-grease-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses an unknown condition operator. SDKs must remove this flag rather than falling through to the catch-all allocation or rejecting the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": [ + { + "key": "grease-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": "NOT_A_REAL_OPERATOR", + "value": "anything" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-regex-flag": { + "key": "invalid-regex-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses the invalid regex *@datadoghq.com. SDKs must remove this flag rather than falling through to a later allocation or rejecting the whole config.", + "variations": { + "invalid-regex-match": { + "key": "invalid-regex-match", + "value": "invalid-regex-match" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-regex-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "email", + "operator": "MATCHES", + "value": "*@datadoghq.com" + } + ] + } + ], + "splits": [ + { + "variationKey": "invalid-regex-match", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-regex-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "variant-type-mismatch-flag": { + "key": "variant-type-mismatch-flag", + "enabled": true, + "variationType": "INTEGER", + "comment": "This flag is intentionally invalid: one variation contains a string even though the flag declares INTEGER values. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "invalid": { + "key": "invalid", + "value": "not-an-integer" + }, + "valid": { + "key": "valid", + "value": 0 + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "invalid", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-gt-condition-flag": { + "key": "invalid-gt-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives GT a string operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-gt-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "age", + "operator": "GT", + "value": "not-a-number" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-gt-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-one-of-condition-flag": { + "key": "invalid-one-of-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives ONE_OF a scalar operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-one-of-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "role", + "operator": "ONE_OF", + "value": "admin" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-one-of-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "invalid-is-null-condition-flag": { + "key": "invalid-is-null-condition-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately gives IS_NULL a non-boolean operand. SDKs must remove this flag rather than falling through to the catch-all allocation.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "invalid-is-null-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "deleted_at", + "operator": "IS_NULL", + "value": "not-a-boolean" + } + ] + } + ], + "splits": [ + { + "variationKey": "trap", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "invalid-is-null-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "malformed-allocations-flag": { + "key": "malformed-allocations-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: allocations must be an array. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "on": { + "key": "on", + "value": "on" + } + }, + "allocations": "this-is-not-a-list" + }, + "missing-split-shards-flag": { + "key": "missing-split-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards is required even when the intended split is unsharded. SDKs must treat omitted shards differently from an explicit empty shards array.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-split-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap" + } + ], + "doLog": true + } + ] + }, + "overflow-total-shards-flag": { + "key": "overflow-total-shards-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].totalShards must fit in an unsigned 32-bit integer. SDKs must remove this flag rather than overflowing or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "overflow-total-shards-flag", + "totalShards": 4294967296, + "ranges": [ + { + "start": 0, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "negative-shard-range-start-flag": { + "key": "negative-shard-range-start-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[].start must be an unsigned integer. SDKs must remove this flag rather than treating a negative range as matching traffic.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + } + }, + "allocations": [ + { + "key": "malformed-shard-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "negative-shard-range-start-flag", + "totalShards": 100, + "ranges": [ + { + "start": -1, + "end": 1 + } + ] + } + ] + } + ], + "doLog": true + } + ] + }, + "null-shard-range-flag": { + "key": "null-shard-range-flag", + "enabled": true, + "variationType": "STRING", + "comment": "This flag is intentionally UFC-schema-invalid: split.shards[].ranges[] contains null. SDKs must remove this flag rather than panicking or poisoning the whole config.", + "variations": { + "trap": { + "key": "trap", + "value": "trap" + }, + "catch-all": { + "key": "catch-all", + "value": "catch-all" + } + }, + "allocations": [ + { + "key": "null-shard-range-allocation", + "rules": [], + "splits": [ + { + "variationKey": "trap", + "shards": [ + { + "salt": "null-shard-range-flag", + "totalShards": 8192, + "ranges": [ + null + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "null-shard-range-catch-all", + "rules": [], + "splits": [ + { + "variationKey": "catch-all", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "valid-flag-after-invalid-config": { + "key": "valid-flag-after-invalid-config", + "enabled": true, + "variationType": "STRING", + "comment": "This valid neighbor proves malformed or unsupported flags do not poison the rest of the remote config.", + "variations": { + "expected": { + "key": "expected", + "value": "expected" + } + }, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "expected", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-precedence-test": { + "key": "semver-precedence-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates SemVer precedence rules that are easy to get wrong: multi-digit numeric prerelease ordering (beta.2 < beta.11), build-metadata-ignored across every operator (EQ/NEQ/LT/LTE/GT/GTE), and alphanumeric prerelease ordering (alpha < alpha.1 < alpha.beta < beta < release). Each allocation gates on a unique SEMVER_EQ attribute version so test attributes cannot cross-match other allocations; negative build-metadata cases (NEQ/LT/GT) fall through to DEFAULT.", + "variations": { + "numeric-prerelease": { + "key": "numeric-prerelease", + "value": "numeric-prerelease" + }, + "build-equal-precedence": { + "key": "build-equal-precedence", + "value": "build-equal-precedence" + }, + "build-dotted-eq": { + "key": "build-dotted-eq", + "value": "build-dotted-eq" + }, + "build-lte": { + "key": "build-lte", + "value": "build-lte" + }, + "build-gte": { + "key": "build-gte", + "value": "build-gte" + }, + "alpha-before-alpha-1": { + "key": "alpha-before-alpha-1", + "value": "alpha-before-alpha-1" + }, + "alpha-1-before-alpha-beta": { + "key": "alpha-1-before-alpha-beta", + "value": "alpha-1-before-alpha-beta" + }, + "alpha-beta-before-beta": { + "key": "alpha-beta-before-beta", + "value": "alpha-beta-before-beta" + }, + "beta-before-release": { + "key": "beta-before-release", + "value": "beta-before-release" + }, + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "numeric-prerelease", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "10.0.0-beta.2" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "10.0.0-beta.11" + } + ] + } + ], + "splits": [ + { + "variationKey": "numeric-prerelease", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-equal-precedence", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+linux" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "20.0.0+darwin" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-equal-precedence", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-dotted-eq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0+exp.sha.5114f85" + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "30.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-dotted-eq", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "40.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LTE", + "value": "40.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-lte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gte", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "50.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GTE", + "value": "50.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "build-gte", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-neq", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "60.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_NEQ", + "value": "60.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-lt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "70.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "70.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "build-gt", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "80.0.0+build.42" + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "80.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-before-alpha-1", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "90.0.0-alpha" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "90.0.0-alpha.1" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-before-alpha-1", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-1-before-alpha-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "91.0.0-alpha.1" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "91.0.0-alpha.beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-1-before-alpha-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "alpha-beta-before-beta", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "92.0.0-alpha.beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "92.0.0-beta" + } + ] + } + ], + "splits": [ + { + "variationKey": "alpha-beta-before-beta", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "beta-before-release", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "93.0.0-beta" + }, + { + "attribute": "app_version", + "operator": "SEMVER_LT", + "value": "93.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "beta-before-release", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-input-test": { + "key": "semver-version-parts-input-test", + "enabled": true, + "variationType": "STRING", + "comment": "Exercises one- through five-part runtime attributes using only canonical three-part configured comparands, so comparand ingestion cannot prevent an input case from running. Scenario gates isolate each runtime comparison.", + "variations": { + "one-part-input": { + "key": "one-part-input", + "value": "one-part-input" + }, + "two-part-input": { + "key": "two-part-input", + "value": "two-part-input" + }, + "four-part-input": { + "key": "four-part-input", + "value": "four-part-input" + }, + "five-part-input": { + "key": "five-part-input", + "value": "five-part-input" + } + }, + "allocations": [ + { + "key": "one-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "one-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "two-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "two-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "four-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "four-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-input", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "five-part-input", + "rules": [ + { + "conditions": [ + { + "attribute": "scenario", + "operator": "ONE_OF", + "value": [ + "five-part-input" + ] + }, + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "17.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-input", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-one-part-comparand-test": { + "key": "semver-version-parts-one-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a one-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "one-part-comparand": { + "key": "one-part-comparand", + "value": "one-part-comparand" + } + }, + "allocations": [ + { + "key": "one-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18" + } + ] + } + ], + "splits": [ + { + "variationKey": "one-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-two-part-comparand-test": { + "key": "semver-version-parts-two-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a two-part configured SemVer comparand from every other extended comparand shape.", + "variations": { + "two-part-comparand": { + "key": "two-part-comparand", + "value": "two-part-comparand" + } + }, + "allocations": [ + { + "key": "two-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "18.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "two-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-four-part-comparand-test": { + "key": "semver-version-parts-four-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a four-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "four-part-comparand": { + "key": "four-part-comparand", + "value": "four-part-comparand" + } + }, + "allocations": [ + { + "key": "four-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "four-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-version-parts-five-part-comparand-test": { + "key": "semver-version-parts-five-part-comparand-test", + "enabled": true, + "variationType": "STRING", + "comment": "Isolates a five-part configured SemVer comparand while the runtime attribute remains a canonical three-part version.", + "variations": { + "five-part-comparand": { + "key": "five-part-comparand", + "value": "five-part-comparand" + } + }, + "allocations": [ + { + "key": "five-part-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_GT", + "value": "18.0.0.0.0" + } + ] + } + ], + "splits": [ + { + "variationKey": "five-part-comparand", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-syntax-test": { + "key": "semver-invalid-comparand-syntax-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-parseable SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "not-a-version" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-valid-comparand-hyphenated-prerelease-test": { + "key": "semver-valid-comparand-hyphenated-prerelease-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag uses a valid SemVer comparand with repeated hyphens inside its prerelease identifiers.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "valid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3----R-S.12.9.1--.12+meta" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-consecutive-dots-test": { + "key": "semver-invalid-comparand-consecutive-dots-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses consecutive dots in its SemVer prerelease comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.0.0-alpha..1" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-dot-suffix-test": { + "key": "semver-invalid-comparand-dot-suffix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a Ruby-style dot-suffixed SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "1.2.3.DEV" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-vprefix-test": { + "key": "semver-invalid-comparand-vprefix-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a v-prefixed SemVer comparand, which is not valid SemVer. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "v1.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-leading-zero-test": { + "key": "semver-invalid-comparand-leading-zero-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a leading-zero core component in its SemVer comparand. SDKs must remove this flag without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": "01.2.3" + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-leaf-on": { + "key": "dependent-leaf-on", + "enabled": true, + "variationType": "STRING", + "variations": { + "on": { + "key": "on", + "value": "leaf-on" + } + }, + "allocations": [ + { + "key": "leaf-static", + "splits": [ + { + "variationKey": "on", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-direct-root": { + "key": "dependent-direct-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "direct-match" + }, + "fallback": { + "key": "fallback", + "value": "direct-fallback" + } + }, + "allocations": [ + { + "key": "dependent-match", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-one-of-miss-root": { + "key": "dependent-one-of-miss-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "one-of-fallback" + } + }, + "allocations": [ + { + "key": "variant-not-in-list", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "off" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-not-one-of-hit-root": { + "key": "dependent-not-one-of-hit-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "not-one-of-fallback" + } + }, + "allocations": [ + { + "key": "variant-in-excluded-list", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "NOT_ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-transitive-parent": { + "key": "dependent-transitive-parent", + "enabled": true, + "variationType": "STRING", + "variations": { + "parent-on": { + "key": "parent-on", + "value": "parent-on" + }, + "parent-off": { + "key": "parent-off", + "value": "parent-off" + } + }, + "allocations": [ + { + "key": "leaf-on", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "parent-on", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "parent-off", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-transitive-root": { + "key": "dependent-transitive-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "transitive-match" + }, + "fallback": { + "key": "fallback", + "value": "transitive-fallback" + } + }, + "allocations": [ + { + "key": "parent-match", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-transitive-parent" + }, + "operator": "ONE_OF", + "value": [ + "parent-on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-depth-three-parent": { + "key": "dependent-depth-three-parent", + "enabled": true, + "variationType": "STRING", + "variations": { + "ready": { + "key": "ready", + "value": "ready" + } + }, + "allocations": [ + { + "key": "transitive-parent-ready", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-transitive-parent" + }, + "operator": "ONE_OF", + "value": [ + "parent-on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "ready", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-depth-three-root": { + "key": "dependent-depth-three-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "over-depth", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-depth-three-parent" + }, + "operator": "ONE_OF", + "value": [ + "ready" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-short-circuit-root": { + "key": "dependent-short-circuit-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + }, + "fallback": { + "key": "fallback", + "value": "short-circuit-fallback" + } + }, + "allocations": [ + { + "key": "short-circuited-rule", + "rules": [ + { + "conditions": [ + { + "attribute": "gate", + "operator": "ONE_OF", + "value": [ + "open" + ] + }, + { + "flagEvaluation": { + "key": "dependent-missing-prerequisite" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + }, + { + "key": "fallback", + "splits": [ + { + "variationKey": "fallback", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-memoized-root": { + "key": "dependent-memoized-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "memoized-match" + } + }, + "allocations": [ + { + "key": "same-prerequisite-twice", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "NOT_ONE_OF", + "value": [ + "off" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-atomic-error-root": { + "key": "dependent-atomic-error-root", + "enabled": true, + "variationType": "STRING", + "variations": { + "matched": { + "key": "matched", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "present-then-missing", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-leaf-on" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + }, + { + "flagEvaluation": { + "key": "dependent-missing-prerequisite" + }, + "operator": "ONE_OF", + "value": [ + "on" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-cycle-a": { + "key": "dependent-cycle-a", + "enabled": true, + "variationType": "STRING", + "variations": { + "a": { + "key": "a", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "a-needs-b", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-cycle-b" + }, + "operator": "ONE_OF", + "value": [ + "b" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "a", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "dependent-cycle-b": { + "key": "dependent-cycle-b", + "enabled": true, + "variationType": "STRING", + "variations": { + "b": { + "key": "b", + "value": "must-not-be-returned" + } + }, + "allocations": [ + { + "key": "b-needs-a", + "rules": [ + { + "conditions": [ + { + "flagEvaluation": { + "key": "dependent-cycle-a" + }, + "operator": "ONE_OF", + "value": [ + "a" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "b", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "semver-invalid-comparand-non-string-test": { + "key": "semver-invalid-comparand-non-string-test", + "enabled": true, + "variationType": "STRING", + "comment": "This flag deliberately uses a non-string (JSON number) SemVer comparand. SDKs must reject this as an invalid comparand without rejecting the full configuration.", + "variations": { + "matched": { + "key": "matched", + "value": "matched" + } + }, + "allocations": [ + { + "key": "invalid-comparand", + "rules": [ + { + "conditions": [ + { + "attribute": "app_version", + "operator": "SEMVER_EQ", + "value": 1.2 + } + ] + } + ], + "splits": [ + { + "variationKey": "matched", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-basic-flag": { + "key": "serial-id-basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 42 + } + ], + "doLog": true + } + ] + }, + "serial-id-zero-flag": { + "key": "serial-id-zero-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "zero-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 0 + } + ], + "doLog": true + } + ] + }, + "serial-id-missing-flag": { + "key": "serial-id-missing-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "no-serial-allocation", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [] + } + ], + "doLog": true + } + ] + }, + "serial-id-targeting-flag": { + "key": "serial-id-targeting-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "vip": { + "key": "vip", + "value": "vip" + }, + "standard": { + "key": "standard", + "value": "standard" + } + }, + "allocations": [ + { + "key": "vip-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "segment", + "operator": "ONE_OF", + "value": [ + "vip" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "vip", + "shards": [], + "serialId": 201 + } + ], + "doLog": true + }, + { + "key": "default-allocation", + "rules": [], + "splits": [ + { + "variationKey": "standard", + "shards": [], + "serialId": 200 + } + ], + "doLog": true + } + ] + }, + "basic-flag": { + "key": "basic-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "on-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 100 + } + ], + "doLog": true + } + ] + }, + "experiment-flag": { + "key": "experiment-flag", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "control" + }, + "treatment": { + "key": "treatment", + "value": "treatment" + } + }, + "allocations": [ + { + "key": "experiment-allocation", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 108 + } + ], + "doLog": true + } + ] + }, + "no-log-flag": { + "key": "no-log-flag", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "on": { + "key": "on", + "value": true + }, + "off": { + "key": "off", + "value": false + } + }, + "allocations": [ + { + "key": "no-log-allocation", + "rules": [], + "splits": [ + { + "variationKey": "on", + "shards": [], + "serialId": 200 + } + ], + "doLog": false + } + ] + }, + "multi-serial-flag-1": { + "key": "multi-serial-flag-1", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v1": { + "key": "v1", + "value": 1 + } + }, + "allocations": [ + { + "key": "allocation-1", + "rules": [], + "splits": [ + { + "variationKey": "v1", + "shards": [], + "serialId": 128 + } + ], + "doLog": true + } + ] + }, + "multi-serial-flag-2": { + "key": "multi-serial-flag-2", + "enabled": true, + "variationType": "INTEGER", + "variations": { + "v2": { + "key": "v2", + "value": 2 + } + }, + "allocations": [ + { + "key": "allocation-2", + "rules": [], + "splits": [ + { + "variationKey": "v2", + "shards": [], + "serialId": 130 + } + ], + "doLog": true + } + ] } } -} \ No newline at end of file +} diff --git a/tests/parametric/test_ffe/serial-id-metadata-flags.json b/tests/parametric/test_ffe/serial-id-metadata-flags.json deleted file mode 100644 index 35bf0ab4c74..00000000000 --- a/tests/parametric/test_ffe/serial-id-metadata-flags.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "createdAt": "2024-01-01T00:00:00.000Z", - "format": "SERVER", - "environment": { "name": "Test" }, - "flags": { - "serial-id-basic-flag": { - "key": "serial-id-basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 42 } - ], - "doLog": true - } - ] - }, - "serial-id-zero-flag": { - "key": "serial-id-zero-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { "key": "on", "value": true }, - "off": { "key": "off", "value": false } - }, - "allocations": [ - { - "key": "zero-allocation", - "rules": [], - "splits": [ - { "variationKey": "on", "shards": [], "serialId": 0 } - ], - "doLog": true - } - ] - }, - "serial-id-missing-flag": { - "key": "serial-id-missing-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { "key": "control", "value": "control" }, - "treatment": { "key": "treatment", "value": "treatment" } - }, - "allocations": [ - { - "key": "no-serial-allocation", - "rules": [], - "splits": [ - { "variationKey": "control", "shards": [] } - ], - "doLog": true - } - ] - }, - "serial-id-targeting-flag": { - "key": "serial-id-targeting-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "vip": { "key": "vip", "value": "vip" }, - "standard": { "key": "standard", "value": "standard" } - }, - "allocations": [ - { - "key": "vip-allocation", - "rules": [ - { - "conditions": [ - { "attribute": "segment", "operator": "ONE_OF", "value": ["vip"] } - ] - } - ], - "splits": [ - { "variationKey": "vip", "shards": [], "serialId": 201 } - ], - "doLog": true - }, - { - "key": "default-allocation", - "rules": [], - "splits": [ - { "variationKey": "standard", "shards": [], "serialId": 200 } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/span-enrichment-flags.json b/tests/parametric/test_ffe/span-enrichment-flags.json deleted file mode 100644 index 232315bcbe8..00000000000 --- a/tests/parametric/test_ffe/span-enrichment-flags.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "createdAt": "2024-04-17T19:40:53.716Z", - "format": "SERVER", - "environment": { - "name": "Test" - }, - "flags": { - "basic-flag": { - "key": "basic-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "on-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 100 - } - ], - "doLog": true - } - ] - }, - "experiment-flag": { - "key": "experiment-flag", - "enabled": true, - "variationType": "STRING", - "variations": { - "control": { - "key": "control", - "value": "control" - }, - "treatment": { - "key": "treatment", - "value": "treatment" - } - }, - "allocations": [ - { - "key": "experiment-allocation", - "rules": [], - "splits": [ - { - "variationKey": "treatment", - "shards": [], - "serialId": 108 - } - ], - "doLog": true - } - ] - }, - "no-log-flag": { - "key": "no-log-flag", - "enabled": true, - "variationType": "BOOLEAN", - "variations": { - "on": { - "key": "on", - "value": true - }, - "off": { - "key": "off", - "value": false - } - }, - "allocations": [ - { - "key": "no-log-allocation", - "rules": [], - "splits": [ - { - "variationKey": "on", - "shards": [], - "serialId": 200 - } - ], - "doLog": false - } - ] - }, - "multi-serial-flag-1": { - "key": "multi-serial-flag-1", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v1": { - "key": "v1", - "value": 1 - } - }, - "allocations": [ - { - "key": "allocation-1", - "rules": [], - "splits": [ - { - "variationKey": "v1", - "shards": [], - "serialId": 128 - } - ], - "doLog": true - } - ] - }, - "multi-serial-flag-2": { - "key": "multi-serial-flag-2", - "enabled": true, - "variationType": "INTEGER", - "variations": { - "v2": { - "key": "v2", - "value": 2 - } - }, - "allocations": [ - { - "key": "allocation-2", - "rules": [], - "splits": [ - { - "variationKey": "v2", - "shards": [], - "serialId": 130 - } - ], - "doLog": true - } - ] - } - } -} diff --git a/tests/parametric/test_ffe/test-case-regex-flag.json b/tests/parametric/test_ffe/test-case-regex-flag.json deleted file mode 100644 index 94aa87f23a9..00000000000 --- a/tests/parametric/test_ffe/test-case-regex-flag.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "alice", - "attributes": { - "version": "1.15.0", - "email": "alice@example.com" - }, - "result": { - "value": "partial-example" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "bob", - "attributes": { - "version": "0.20.1", - "email": "bob@test.com" - }, - "result": { - "value": "test" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "charlie", - "attributes": { - "version": "2.1.13" - }, - "result": { - "value": "none" - } - }, - { - "flag": "regex-flag", - "variationType": "STRING", - "defaultValue": "none", - "targetingKey": "derek", - "attributes": { - "version": "2.1.13", - "email": "derek@gmail.com" - }, - "result": { - "value": "none" - } - } -] diff --git a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json b/tests/parametric/test_ffe/test-flag-that-does-not-exist.json deleted file mode 100644 index 7499bba1c50..00000000000 --- a/tests/parametric/test_ffe/test-flag-that-does-not-exist.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "alice", - "attributes": { - "email": "alice@mycompany.com", - "country": "US" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "bob", - "attributes": { - "email": "bob@example.com", - "country": "Canada" - }, - "result": { - "value": 0.0 - } - }, - { - "flag": "flag-that-does-not-exist", - "variationType": "NUMERIC", - "defaultValue": 0.0, - "targetingKey": "charlie", - "attributes": { - "age": 50 - }, - "result": { - "value": 0.0 - } - } -] diff --git a/tests/parametric/test_ffe/test_configuration_sources.py b/tests/parametric/test_ffe/test_configuration_sources.py index 79a4a716b8f..ca533dc42bd 100644 --- a/tests/parametric/test_ffe/test_configuration_sources.py +++ b/tests/parametric/test_ffe/test_configuration_sources.py @@ -34,7 +34,7 @@ parametrize = pytest.mark.parametrize pytest_plugins = ["utils.mocked_backend.ffe"] -UFC_VALID_FIXTURE = Path(__file__).parent / "flags-v1.json" +UFC_VALID_FIXTURE = Path(__file__).parent / "ffe-system-test-data" / "ufc-config.json" RC_PRODUCT = "FFE_FLAGS" TEST_API_KEY = "system-tests-mock-api-key" MOCK_STATUS_ATTEMPTS = 25 diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 35b15215c50..c9138cbe6d9 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -18,6 +18,8 @@ RC_PATH = f"datadog/2/{RC_PRODUCT}" FFE_READY_RETRY_ATTEMPTS = 10 FFE_READY_RETRY_INTERVAL_SECONDS = 0.2 +FFE_SYSTEM_TEST_DATA_DIR = Path(__file__).parent / "ffe-system-test-data" +FFE_EVALUATION_CASES_DIR = FFE_SYSTEM_TEST_DATA_DIR / "evaluation-cases" parametrize = pytest.mark.parametrize @@ -25,24 +27,18 @@ # Load the UFC fixture file at module level def _load_ufc_fixture() -> dict[str, Any]: """Load the UFC fixture file.""" - fixture_path = Path(__file__).parent / "flags-v1.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") + fixture_path = FFE_SYSTEM_TEST_DATA_DIR / "ufc-config.json" with fixture_path.open() as f: return json.load(f) def _get_test_case_files() -> list[str]: - """Get all test case files from the fixtures directory.""" - test_data_dir = Path(__file__).parent - if not test_data_dir.exists(): - return [] - - # Exclude base fixtures that aren't test cases - excluded = {"flags-v1.json", "serial-id-metadata-flags.json", "span-enrichment-flags.json"} - return [f.name for f in test_data_dir.iterdir() if f.suffix == ".json" and f.name not in excluded] + """Get all canonical evaluation case files.""" + case_files = sorted(path.name for path in FFE_EVALUATION_CASES_DIR.glob("*.json")) + if not case_files: + raise FileNotFoundError(f"No canonical evaluation cases found in {FFE_EVALUATION_CASES_DIR}") + return case_files # Load fixture at module level for reuse across tests @@ -103,7 +99,7 @@ def _ffe_evaluate_with_rc_retry( flag: str, variation_type: str, default_value: bool | str | float | dict[str, Any], - targeting_key: str, + targeting_key: str | None, attributes: dict[str, Any] | None = None, ) -> dict[str, Any]: result = test_library.ffe_evaluate( @@ -149,23 +145,14 @@ def test_ffe_remote_config(self, test_agent: TestAgentAPI, test_library: APMLibr @parametrize("library_env", [{**DEFAULT_ENVVARS}]) @parametrize("test_case_file", ALL_TEST_CASE_FILES) def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: - """Test FFE flag evaluation logic with various targeting scenarios. - - This is the core FFE test that validates the OpenFeature provider correctly: - 1. Loads flag configurations from Remote Config (UFC format) - 2. Evaluates flags based on targeting rules and evaluation context - 3. Returns correct variation values for different variation types - 4. Handles user targeting, attribute matching, and rollout percentages - - """ + """Test every expected result field present in each canonical fixture case.""" # Load the test case file - test_case_path = Path(__file__).parent / test_case_file - - if not test_case_path.exists(): - pytest.skip(f"Test case file not found: {test_case_path}") + test_case_path = FFE_EVALUATION_CASES_DIR / test_case_file with test_case_path.open() as f: test_cases = json.load(f) + assert isinstance(test_cases, list), f"Expected an evaluation-case array in {test_case_file}" + assert test_cases, f"No evaluation cases in {test_case_file}" # Set up UFC Remote Config and wait for it to be applied _set_and_wait_ffe_rc(test_agent, UFC_FIXTURE_DATA) @@ -181,7 +168,7 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI default_value = test_case["defaultValue"] targeting_key = test_case["targetingKey"] attributes = test_case.get("attributes", {}) - expected_result = test_case["result"]["value"] + expected_result = test_case["result"] result = _ffe_evaluate_with_rc_retry( test_library, @@ -195,11 +182,15 @@ def test_ffe_flag_evaluation(self, test_case_file: str, test_agent: TestAgentAPI f"Test case {i} in {test_case_file} failed: FFE provider did not load RC data after " f"{FFE_READY_RETRY_ATTEMPTS} attempts; result={result}" ) - actual_value = result.get("value") - - # Assert the evaluation result matches expected value - assert actual_value == expected_result, ( - f"Test case {i} in {test_case_file} failed: " - f"flag='{flag}', targetingKey='{targeting_key}', " - f"expected={expected_result}, actual={actual_value}" - ) + for field in ("value", "reason", "errorCode"): + if field not in expected_result: + continue + assert result.get(field) == expected_result[field], ( + f"Test case {i} in {test_case_file} failed: " + f"flag='{flag}', targetingKey='{targeting_key}', field='{field}', " + f"expected={expected_result[field]!r}, actual={result.get(field)!r}" + ) + if "errorCode" not in expected_result and expected_result.get("reason") != "ERROR": + assert result.get("errorCode") in {None, ""}, ( + f"Test case {i} in {test_case_file}: unexpected evaluation error; result={result}" + ) diff --git a/tests/parametric/test_ffe/test_serial_id_metadata.py b/tests/parametric/test_ffe/test_serial_id_metadata.py index 8e8a9e14d45..ec0d15b6f3e 100644 --- a/tests/parametric/test_ffe/test_serial_id_metadata.py +++ b/tests/parametric/test_ffe/test_serial_id_metadata.py @@ -23,7 +23,6 @@ import json import time -from pathlib import Path from typing import Any from utils import pytest @@ -32,6 +31,7 @@ from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary +from tests.parametric.test_ffe.utils import load_ufc_fixture_for_cases RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" @@ -43,18 +43,7 @@ parametrize = pytest.mark.parametrize -def _load_serial_id_metadata_fixture() -> dict[str, Any]: - """Load the UFC fixture file for serial ID metadata tests.""" - fixture_path = Path(__file__).parent / "serial-id-metadata-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SERIAL_ID_METADATA_DATA = _load_serial_id_metadata_fixture() +UFC_SERIAL_ID_METADATA_DATA = load_ufc_fixture_for_cases("test-case-serial-id-metadata.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/test_span_enrichment.py b/tests/parametric/test_ffe/test_span_enrichment.py index 6a4e54bdacb..749913bb455 100644 --- a/tests/parametric/test_ffe/test_span_enrichment.py +++ b/tests/parametric/test_ffe/test_span_enrichment.py @@ -21,14 +21,18 @@ import json from utils import pytest -from pathlib import Path from typing import Any from utils import features, scenarios from utils.dd_constants import RemoteConfigApplyState from utils.docker_fixtures import TestAgentAPI from tests.parametric.conftest import APMLibrary -from tests.parametric.test_ffe.utils import decode_delta_varint, encode_delta_varint, hash_targeting_key +from tests.parametric.test_ffe.utils import ( + decode_delta_varint, + encode_delta_varint, + hash_targeting_key, + load_ufc_fixture_for_cases, +) RC_PRODUCT = "FFE_FLAGS" @@ -37,18 +41,7 @@ parametrize = pytest.mark.parametrize -def _load_span_enrichment_fixture() -> dict[str, Any]: - """Load the UFC fixture file for span enrichment tests.""" - fixture_path = Path(__file__).parent / "span-enrichment-flags.json" - - if not fixture_path.exists(): - pytest.skip(f"Fixture file not found: {fixture_path}") - - with fixture_path.open() as f: - return json.load(f) - - -UFC_SPAN_ENRICHMENT_DATA = _load_span_enrichment_fixture() +UFC_SPAN_ENRICHMENT_DATA = load_ufc_fixture_for_cases("test-case-span-enrichment-inputs.json") DEFAULT_ENVVARS = { "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", diff --git a/tests/parametric/test_ffe/utils.py b/tests/parametric/test_ffe/utils.py index 325c825f949..ad655f9cf84 100644 --- a/tests/parametric/test_ffe/utils.py +++ b/tests/parametric/test_ffe/utils.py @@ -11,12 +11,26 @@ import base64 import hashlib +import json +from pathlib import Path +from typing import Any # Varint encoding constants VARINT_DATA_BITS = 0x7F # Lower 7 bits contain data VARINT_CONTINUATION_BIT = 0x80 # MSB indicates more bytes follow +def load_ufc_fixture_for_cases(case_filename: str) -> dict[str, Any]: + """Load the canonical flag definitions used by a specific integration suite.""" + fixture_root = Path(__file__).parent / "ffe-system-test-data" + config = json.loads((fixture_root / "ufc-config.json").read_text()) + cases = json.loads((fixture_root / "evaluation-cases" / case_filename).read_text()) + # Keep unrelated evaluator edge cases out of transport/span integration tests. + flag_keys = dict.fromkeys(case["flag"] for case in cases) + config["flags"] = {key: config["flags"][key] for key in flag_keys} + return config + + def decode_varint(data: bytes, offset: int = 0) -> tuple[int, int]: """Decode a single varint from bytes. diff --git a/tests/test_the_test/test_update_ffe_fixtures.py b/tests/test_the_test/test_update_ffe_fixtures.py new file mode 100644 index 00000000000..8f53acd82ee --- /dev/null +++ b/tests/test_the_test/test_update_ffe_fixtures.py @@ -0,0 +1,145 @@ +# ruff: noqa: SLF001 + +from pathlib import Path +import shutil + +from utils import pytest + +from utils import scenarios +from utils.scripts import update_ffe_fixtures + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["main", "feature/fixture-update", "v1.2.3", "ea8b5cc5"]) +def test_validate_fixture_ref_accepts_git_refs(fixture_ref: str) -> None: + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +@pytest.mark.parametrize("fixture_ref", ["", "-main", "../main", "main^", "main ref"]) +def test_validate_fixture_ref_rejects_unsafe_refs(fixture_ref: str) -> None: + with pytest.raises(ValueError, match="Invalid FFE fixture ref"): + update_ffe_fixtures._validate_fixture_ref(fixture_ref) + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_imports_only_server_evaluator_data(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + (source / "evaluation-cases").mkdir(parents=True) + snapshot.mkdir() + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + (source / "README.md").write_text("upstream documentation", encoding="utf-8") + (source / "AGENTS.md").write_text("upstream instructions", encoding="utf-8") + (source / "precomputed-assignments").mkdir() + + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + assert (snapshot / "ufc-config.json").is_file() + assert (snapshot / "evaluation-cases" / "case.json").is_file() + assert not (snapshot / "README.md").exists() + assert not (snapshot / "AGENTS.md").exists() + assert not (snapshot / "precomputed-assignments").exists() + + +@scenarios.test_the_test +def test_copy_fixture_snapshot_rejects_symbolic_links(tmp_path: Path) -> None: + source = tmp_path / "source" + snapshot = tmp_path / "snapshot" + source.mkdir() + snapshot.mkdir() + (source / "evaluation-cases").mkdir() + (source / "ufc-config.json").write_text("{}", encoding="utf-8") + target = source / "target.json" + target.write_text("{}", encoding="utf-8") + (source / "evaluation-cases" / "linked.json").symlink_to(target) + + with pytest.raises(ValueError, match="Refusing to copy symbolic link"): + update_ffe_fixtures._copy_fixture_snapshot(source, snapshot) + + +@scenarios.test_the_test +def test_validate_fixture_snapshot_counts_evaluations(tmp_path: Path) -> None: + cases_directory = tmp_path / "evaluation-cases" + cases_directory.mkdir() + (tmp_path / "ufc-config.json").write_text('{"flags": {"one": {}}}', encoding="utf-8") + (cases_directory / "first.json").write_text('[{"flag": "one"}]', encoding="utf-8") + (cases_directory / "second.json").write_text('[{"flag": "two"}, {"flag": "three"}]', encoding="utf-8") + + assert update_ffe_fixtures._validate_fixture_snapshot(tmp_path) == 3 + + +@scenarios.test_the_test +def test_have_same_contents_ignores_generated_source_metadata(tmp_path: Path) -> None: + snapshot = tmp_path / "snapshot" + destination = tmp_path / "destination" + snapshot.mkdir() + destination.mkdir() + (snapshot / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "fixture.json").write_text("{}", encoding="utf-8") + (destination / "SOURCE.md").write_text("generated", encoding="utf-8") + + assert update_ffe_fixtures._have_same_contents(snapshot, destination) + + (destination / "fixture.json").write_text('{"changed": true}', encoding="utf-8") + assert not update_ffe_fixtures._have_same_contents(snapshot, destination) + + +@scenarios.test_the_test +def test_update_fixtures_preserves_snapshot_on_validation_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + (source / "evaluation-cases" / "case.json").write_text("[]", encoding="utf-8") + destination.mkdir() + (destination / "SOURCE.md").write_text("original snapshot", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + with pytest.raises(ValueError, match="at least one evaluation case"): + update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_text() == "original snapshot" + + +@scenarios.test_the_test +def test_update_fixtures_records_source_and_removes_stale_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "upstream" + destination = tmp_path / "destination" + output = tmp_path / "outputs" + (source / "evaluation-cases").mkdir(parents=True) + (source / "ufc-config.json").write_text('{"flags": {"flag": {}}}', encoding="utf-8") + case_file = source / "evaluation-cases" / "case.json" + case_file.write_text('[{"flag": "flag"}]', encoding="utf-8") + destination.mkdir() + (destination / "stale.json").write_text("[]", encoding="utf-8") + + def fake_git(working_directory: Path, *arguments: str) -> str: + if arguments[0] == "checkout": + shutil.copytree(source, working_directory, dirs_exist_ok=True) + return "a" * 40 + + monkeypatch.setattr(update_ffe_fixtures, "_run_git", fake_git) + monkeypatch.setattr(update_ffe_fixtures, "FIXTURE_DESTINATION", destination) + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + assert update_ffe_fixtures.update_fixtures("main") + assert not (destination / "stale.json").exists() + assert (destination / "evaluation-cases" / "case.json").read_bytes() == case_file.read_bytes() + metadata = (destination / "SOURCE.md").read_bytes() + assert ("a" * 40).encode() in metadata + assert "fixture_count=1\nchanged=true\n" in output.read_text() + + (source / "README.md").write_text("documentation-only upstream change", encoding="utf-8") + assert not update_ffe_fixtures.update_fixtures("main") + assert (destination / "SOURCE.md").read_bytes() == metadata + assert output.read_text().endswith("changed=false\n") diff --git a/utils/build/docker/nodejs/parametric/server.js b/utils/build/docker/nodejs/parametric/server.js index 78024c9231f..32b021b3e80 100644 --- a/utils/build/docker/nodejs/parametric/server.js +++ b/utils/build/docker/nodejs/parametric/server.js @@ -650,6 +650,7 @@ app.post('/ffe/evaluate', async (req, res) => { res.json({ value: details.value, reason: details.reason, + errorCode: details.errorCode, flagMetadata: details.flagMetadata ?? null }); }); diff --git a/utils/build/docker/ruby/parametric/server.rb b/utils/build/docker/ruby/parametric/server.rb index cf5fed16bb2..0461034fd50 100644 --- a/utils/build/docker/ruby/parametric/server.rb +++ b/utils/build/docker/ruby/parametric/server.rb @@ -869,20 +869,26 @@ def handle_ffe_evaluation(req, res) options = { flag_key: args.flag, default_value: args.default_value, evaluation_context: context } + options[:default_value] = args.default_value.to_f if args.variation_type == 'NUMERIC' - value = + details = case args.variation_type - when 'BOOLEAN'then client.fetch_boolean_value(**options) - when 'STRING' then client.fetch_string_value(**options) - when 'INTEGER' then client.fetch_integer_value(**options) - when 'NUMERIC' then client.fetch_float_value(**options) - when 'JSON' then client.fetch_object_value(**options) - else 'FATAL_UNEXPECTED_VARIATION_TYPE' + when 'BOOLEAN'then client.fetch_boolean_details(**options) + when 'STRING' then client.fetch_string_details(**options) + when 'INTEGER' then client.fetch_integer_details(**options) + when 'NUMERIC' then client.fetch_float_details(**options) + when 'JSON' then client.fetch_object_details(**options) + else raise ArgumentError, "Unexpected variation type: #{args.variation_type}" end - res.write({value: value, reason: 'DEFAULT'}.to_json) + res.write({ + value: details.value, + reason: details.reason, + errorCode: details.error_code, + flagMetadata: details.flag_metadata + }.to_json) rescue => e - res.write({value: args.default_value, reason: 'ERROR'}.to_json) + res.write({value: args.default_value, reason: 'ERROR', errorCode: 'GENERAL', errorMessage: e.message}.to_json) end end diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index 5a512b87c19..895f006e227 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -824,7 +824,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: @@ -1267,7 +1267,7 @@ def ffe_evaluate( flag: str, variation_type: str, default_value: bool | str | float | dict, - targeting_key: str, + targeting_key: str | None, attributes: dict | None = None, span_id: int | str | None = None, ) -> dict: diff --git a/utils/mocked_backend/ffe.py b/utils/mocked_backend/ffe.py index 5b288d176e2..dbb1314a5f4 100644 --- a/utils/mocked_backend/ffe.py +++ b/utils/mocked_backend/ffe.py @@ -48,7 +48,7 @@ EXPECTED_DD_ENV = "test" CONFIG_QUERY = urlencode({"dd_env": EXPECTED_DD_ENV}) REPO_ROOT = Path(__file__).parents[2] -UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "flags-v1.json" +UFC_FIXTURE_PATH = REPO_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" / "ufc-config.json" MALFORMED_UFC_BYTES = b'{"flags": [' UFC_RESPONSE_TYPE = "universal-flag-configuration" diff --git a/utils/scripts/update_ffe_fixtures.py b/utils/scripts/update_ffe_fixtures.py new file mode 100644 index 00000000000..e7f3b95f484 --- /dev/null +++ b/utils/scripts/update_ffe_fixtures.py @@ -0,0 +1,206 @@ +"""Refresh the checked-in FFE fixture snapshot from its canonical repository.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DESTINATION = REPOSITORY_ROOT / "tests" / "parametric" / "test_ffe" / "ffe-system-test-data" +FIXTURE_REPOSITORY = "https://github.com/DataDog/ffe-system-test-data.git" +FIXTURE_REF_PATTERN = re.compile(r"[A-Za-z0-9._/-]+") + + +def _validate_fixture_ref(fixture_ref: str) -> None: + if ( + not fixture_ref + or fixture_ref.startswith("-") + or ".." in fixture_ref + or FIXTURE_REF_PATTERN.fullmatch(fixture_ref) is None + ): + msg = f"Invalid FFE fixture ref: {fixture_ref!r}" + raise ValueError(msg) + + +def _run_git(working_directory: Path, *arguments: str) -> str: + environment = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + } + result = subprocess.run( + ["git", *arguments], + cwd=working_directory, + env=environment, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _copy_entry(source: Path, destination: Path) -> None: + if source.is_symlink(): + msg = f"Refusing to copy symbolic link from FFE fixture repository: {source}" + raise ValueError(msg) + + if source.is_file(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return + + if not source.is_dir(): + msg = f"Refusing to copy unsupported filesystem entry from FFE fixture repository: {source}" + raise ValueError(msg) + + destination.mkdir(parents=True, exist_ok=True) + for child in sorted(source.iterdir()): + _copy_entry(child, destination / child.name) + + +def _copy_fixture_snapshot(source: Path, snapshot: Path) -> None: + # Import only the server evaluator corpus that this suite consumes. Other + # upstream suites (for example precomputed assignments) have different formats. + for name in ("ufc-config.json", "evaluation-cases"): + _copy_entry(source / name, snapshot / name) + + +def _validate_fixture_snapshot(snapshot: Path) -> int: + config_path = snapshot / "ufc-config.json" + cases_directory = snapshot / "evaluation-cases" + if not config_path.is_file() or not cases_directory.is_dir(): + msg = "FFE fixture repository does not contain the expected fixture layout" + raise ValueError(msg) + + with config_path.open(encoding="utf-8") as config_file: + config = json.load(config_file) + if not isinstance(config, dict) or not isinstance(config.get("flags"), dict) or not config["flags"]: + msg = "FFE UFC fixture must contain a non-empty flags object" + raise ValueError(msg) + + case_files = sorted(cases_directory.glob("*.json")) + if any(not path.is_file() or path.suffix != ".json" for path in cases_directory.iterdir()): + msg = "FFE evaluation-cases must contain only JSON files" + raise ValueError(msg) + if not case_files: + msg = "No FFE JSON fixture files found" + raise ValueError(msg) + + fixture_count = 0 + for case_file in case_files: + with case_file.open(encoding="utf-8") as fixture_file: + test_cases = json.load(fixture_file) + if not isinstance(test_cases, list): + msg = f"{case_file} must contain a JSON array of test cases" + raise TypeError(msg) + if not test_cases: + msg = f"{case_file} must contain at least one evaluation case" + raise ValueError(msg) + fixture_count += len(test_cases) + + if fixture_count == 0: + msg = "No FFE fixture test cases found" + raise ValueError(msg) + + return fixture_count + + +def _relative_files(directory: Path, excluded_files: frozenset[str] = frozenset()) -> list[Path]: + return sorted( + path.relative_to(directory) + for path in directory.rglob("*") + if path.is_file() and path.relative_to(directory).as_posix() not in excluded_files + ) + + +def _have_same_contents(snapshot: Path, destination: Path) -> bool: + if not destination.is_dir(): + return False + + snapshot_files = _relative_files(snapshot) + destination_files = _relative_files(destination, frozenset({"SOURCE.md"})) + if snapshot_files != destination_files: + return False + + return all( + (snapshot / relative_path).read_bytes() == (destination / relative_path).read_bytes() + for relative_path in snapshot_files + ) + + +def _write_source_metadata(snapshot: Path, source_commit: str) -> None: + (snapshot / "SOURCE.md").write_text( + f"""# FFE Fixture Snapshot + +These files are copied from the canonical FFE fixture repository. + +Canonical source: https://github.com/DataDog/ffe-system-test-data +Source commit: {source_commit} + +Do not edit these fixtures directly in system-tests. Add or update shared FFE behavior in +ffe-system-test-data first, then refresh this snapshot. + +The weekly update workflow runs `python3 utils/scripts/update_ffe_fixtures.py` and opens a signed +draft test PR only when the server evaluator fixture contents change. +""", + encoding="utf-8", + ) + + +def _set_github_outputs(source_commit: str, fixture_count: int, *, changed: bool) -> None: + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with Path(github_output).open("a", encoding="utf-8") as output_file: + output_file.write(f"source_commit={source_commit}\n") + output_file.write(f"fixture_count={fixture_count}\n") + output_file.write(f"changed={str(changed).lower()}\n") + + +def update_fixtures(fixture_ref: str) -> bool: + """Update the snapshot and return whether its allowed contents changed.""" + _validate_fixture_ref(fixture_ref) + + with tempfile.TemporaryDirectory(prefix="ffe-system-test-data-") as temporary_directory: + temporary_path = Path(temporary_directory) + source = temporary_path / "source" + snapshot = temporary_path / "snapshot" + source.mkdir() + snapshot.mkdir() + + _run_git(source, "init", "--quiet") + _run_git(source, "remote", "add", "origin", FIXTURE_REPOSITORY) + _run_git(source, "fetch", "--quiet", "--depth", "1", "origin", fixture_ref) + _run_git(source, "checkout", "--quiet", "--detach", "FETCH_HEAD") + source_commit = _run_git(source, "rev-parse", "HEAD") + + _copy_fixture_snapshot(source, snapshot) + fixture_count = _validate_fixture_snapshot(snapshot) + changed = not _have_same_contents(snapshot, FIXTURE_DESTINATION) + + if changed: + _write_source_metadata(snapshot, source_commit) + if FIXTURE_DESTINATION.exists(): + shutil.rmtree(FIXTURE_DESTINATION) + shutil.copytree(snapshot, FIXTURE_DESTINATION) + + _set_github_outputs(source_commit, fixture_count, changed=changed) + print(f"Checked FFE fixtures from DataDog/ffe-system-test-data@{source_commit}") + print(f"Loaded {fixture_count} canonical fixture cases") + print(f"Fixture snapshot changed: {str(changed).lower()}") + return changed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", default="main", help="Branch, tag, or commit from DataDog/ffe-system-test-data") + arguments = parser.parse_args() + update_fixtures(arguments.ref) + + +if __name__ == "__main__": + main()