From a2ce1b3751e7cf1165a0fb7b8cfa44de0a918dfd Mon Sep 17 00:00:00 2001 From: Andy White Date: Wed, 23 Sep 2026 15:46:31 -0600 Subject: [PATCH 1/6] feat: support saved group references v2 --- CONTRIBUTING.md | 306 +++- README.md | 34 + growthbook/core.py | 142 +- tests/cases.json | 1760 ++++++++++++++++++++++- tests/scripts/check_corpus_freshness.py | 21 +- tests/test_corpus_freshness.py | 37 + tests/test_saved_groups.py | 245 ++++ 7 files changed, 2493 insertions(+), 52 deletions(-) create mode 100644 tests/test_corpus_freshness.py create mode 100644 tests/test_saved_groups.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 658c4be..0c31fe8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,34 +2,316 @@ We welcome all contributions! -## Type checking +This repo is the official GrowthBook SDK for Python — a client library for +evaluating feature flags and running experiments, with synchronous and async +clients. -We use MyPy for type checking. +## Requirements + +- **Python 3.9+**, as declared in `pyproject.toml` and `setup.py`. CI tests + 3.9–3.13; the examples below use 3.13. Keep library code compatible with 3.9. +- **Make** for the convenience commands below. You can also invoke `pytest`, + `mypy`, and `flake8` directly from the activated environment. +- **Node.js and npm** only for the optional local Pyright checks. + +`uv` is optional. There is no dependency lockfile; CI installs from +`requirements.txt` and `requirements_dev.txt`, then installs the package. + +## Getting started + +Fork the repo, or clone directly if you have write access: + +```bash +git clone git@github.com:growthbook/growthbook-python.git +cd growthbook-python +``` + +Create a virtual environment and install the package in editable mode with +the development dependencies: ```bash +python3.13 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -e . -r requirements.txt -r requirements_dev.txt +make test make type-check ``` -## Tests +If you use `uv`, the equivalent setup is: + +```bash +uv venv --python 3.13 .venv +source .venv/bin/activate +uv pip install -e . -r requirements.txt -r requirements_dev.txt +``` + +Activate `.venv` in each new terminal with `source .venv/bin/activate`. Configure +your editor to use `.venv/bin/python`. Changes to `growthbook/` are available +immediately without reinstalling the package. + +The unit tests do not need GrowthBook credentials or a running server. Get a +passing test and type-check baseline before changing code. + +## Writing code + +### Layout + +| Path | Contents | +| --- | --- | +| `growthbook/core.py` | Shared feature and experiment evaluation, targeting conditions, hashing, and bucketing | +| `growthbook/growthbook.py` | The sync `GrowthBook` client, feature repository, caching, SSE, and decryption | +| `growthbook/growthbook_client.py` | The async `GrowthBookClient`, shared feature repository, refreshes, and per-user evaluation | +| `growthbook/common_types.py` | Configuration, user and evaluation contexts, results, callback protocols, and sticky bucket interfaces | +| `growthbook/plugins/` | Tracking and request-context plugins | +| `growthbook/codegen.py` | Generator for typed clients built from feature definitions | +| `growthbook/__init__.py`, `growthbook/py.typed` | Public exports, package version, and the typed-package marker | +| `tests/test_*.py`, `tests/conftest.py` | Tests and shared fixtures | +| `tests/cases.json` | Cross-SDK conformance corpus, including Python-local cases | +| `tests/typing/`, `tests/codegen/` | Typing regression cases and code-generation fixtures | +| `tests/scripts/` | Corpus checks, benchmarks, and integration verification scripts | + +### Public API and typing + +`growthbook/__init__.py` lists the package's public exports in `__all__`. +Callers also use the documented client methods and configuration types. Discuss +changes to signatures, defaults, or return types in an issue before implementing +them; type-checking behavior is part of the API too. + +The package ships inline type annotations through `py.typed`. Add annotations +to new library functions and keep public types importable and resolvable at +runtime. When changing typing behavior, update the positive and negative cases +in `tests/typing/` and run both MyPy and Pyright — see +[Type checking](#type-checking). + +The Python 3.9 floor applies even if your local interpreter is newer. Use +`typing_extensions` where a newer typing feature needs a backport. Changes to +the minimum Python version should be discussed explicitly. + +### SDK parity + +GrowthBook SDKs should make identical decisions given identical inputs. The +[JavaScript SDK](https://github.com/growthbook/growthbook/tree/main/packages/sdk-js) +is the reference implementation. When changing evaluation, compare the +corresponding JS behavior, especially hashing, missing values, numeric types, +and targeting comparisons where Python semantics differ. + +Describe deliberate divergences in the PR. A change to which users match a +targeting rule can change live audiences when an application upgrades. -Run the test suite with +### Sync, async, and shared state + +`GrowthBook` stores user attributes on the instance; web applications normally +create one per request and call `destroy()` afterward. `GrowthBookClient` is +reused across requests and receives a separate `UserContext` for each user. +Call `await client.close()` at shutdown, or use its async context manager. + +Both clients use `core.py`, but have separate loading and lifecycle code. +Cover both clients when changing shared behavior. In async code, avoid blocking +the event loop and test cancellation and cleanup of any background work you add. + +Feature repositories and caches can be shared. In particular, +`EnhancedFeatureRepository` instances are keyed by class, API host, and client +key. Keep per-user state isolated, and test with multiple users and clients when +changing cache keys, tracking, or sticky bucketing. The fixtures in +`tests/conftest.py` clean up async tasks and repository singletons between tests; +they do not replace testing the client's own cleanup. + +Use the existing `logging` loggers for diagnostics. Library code should not +configure the application's root logger or print to stdout. + +## Testing ```bash -make test +make test # full suite (pytest) +pytest tests/test_remote_eval.py # one file +pytest tests/test_growthbook.py::test_hash # one parametrized test +pytest -k sticky -x # matching tests, stop on first failure +make coverage # terminal report and HTML, opens a browser +``` + +For coverage without opening a browser: + +```bash +coverage run --source growthbook -m pytest +coverage report -m +coverage html +``` + +Tests use `pytest`, `pytest-mock`, and `pytest-asyncio`. Asyncio runs in strict +mode: mark async tests with `@pytest.mark.asyncio` and use +`@pytest_asyncio.fixture` for async fixtures. Mock external HTTP calls in unit +tests, and close clients and sessions created by the test. + +A bug fix should include a regression test that fails before the fix. For +generator changes, review `tests/codegen/expected_output.py` alongside the +generator and run both `tests/test_codegen.py` and `tests/test_typing.py`. + +### Conformance corpus + +`tests/cases.json` contains cases mirrored from the +[JS corpus](https://github.com/growthbook/growthbook/blob/main/packages/sdk-js/test/cases.json) +plus Python-local extensions and regressions. `tests/test_growthbook.py` loads +the corpus into parametrized tests. Investigate a failing shared case before +changing its expected result. + +CI compares the corpus with live JS `main` on every push: + +```bash +python tests/scripts/check_corpus_freshness.py +``` + +The command needs network access. To compare with a local JS checkout instead: + +```bash +python tests/scripts/check_corpus_freshness.py \ + --js-source /path/to/growthbook/packages/sdk-js/test/cases.json +``` + +Missing cases and changed bodies of shared cases fail the check unless listed +in `tests/scripts/corpus_skiplist.json`. Python-only cases are reported but do +not fail it. Add a reason under `_reasons` for each intentional skip. The check +compares selected corpus sections; it does not replace running the tests. + +Preserve Python-local cases when syncing from upstream. For a parity bug that +affects other SDKs, propose a regression case in the JS corpus as well. + +### Benchmarks and integration checks + +Run benchmarks from the repository root with the virtual environment active: + +```bash +python tests/scripts/benchmark_eval_overhead.py 100000 +python tests/scripts/benchmark_async_client.py +``` + +The first measures sync evaluation overhead; the second measures async +throughput, latency, and event-loop lag with simulated sticky bucket services. +Neither needs external services. Include before/after results in the PR when +changing evaluation performance or concurrency behavior, using the same Python +version, machine, and workload for both runs. + +For remote-evaluation checks against an in-process test proxy: + +```bash +python tests/scripts/verify_remote_eval.py +``` + +This starts a localhost server. Real-proxy checks are separate and need Docker +and GrowthBook credentials; see [the verification scripts](tests/scripts/README.md) +for setup and teardown. + +## Code quality + +### Type checking + +MyPy is pinned in `requirements_dev.txt` and configured in `pyproject.toml`: + +```bash +make type-check +pytest tests/test_typing.py -q ``` -## Linting +The typing regression suite checks valid usage and verifies that invalid calls +fail on the lines marked `# expect-error`. It always runs MyPy and also runs +Pyright when a `pyright` executable is on `PATH`. -Lint the code with flake8 +To reproduce CI's Pyright checks locally, install the version pinned in +`.github/workflows/main.yml`: + +```bash +npm install -g pyright@1.1.403 +pyright +pytest tests/test_typing.py -q +``` + +Keep the virtual environment active so Pyright can resolve the installed +dependencies. CI runs Pyright on Python 3.11; both checker configurations target +Python 3.9 language compatibility. + +### Linting and formatting + +CI's blocking lint check selects syntax and correctness errors. Exclude the +local virtual environment when running it from the repository root: + +```bash +flake8 . --extend-exclude=.venv --count --select=E9,F63,F7,F82 --show-source --statistics +``` + +CI also reports broader style warnings without failing the build: + +```bash +flake8 . --extend-exclude=.venv --count --exit-zero --max-complexity=25 --max-line-length=127 --statistics +``` + +The existing Make target is different: ```bash make lint ``` +It checks only `growthbook/growthbook.py`, uses a 150-character line limit, and +fails on existing style and unused-import issues. A failure there does not by +itself mean the environment is broken. Check whether a finding is introduced +by your change. + +There is no automatic formatter configured. Follow `.editorconfig` and the +surrounding code, use docstrings for declarations that need documentation, and +keep formatting changes limited to the code you touch. + +## Opening pull requests + +Open an issue first for API changes, new configuration, or behavior changes so +the design can be discussed. For a typo or an obvious fix, open a PR directly. + +1. Branch from `main` and keep the diff focused. +2. Add regression coverage and update `README.md` for changes to documented + behavior or usage. +3. Run `make test`, `make type-check`, and the blocking lint command above. + Run Pyright for typing changes, corpus freshness for evaluation or fixture + changes, and benchmarks for performance-sensitive changes. +4. Use descriptive commit subjects with conventional-commit prefixes such as + `fix:`, `feat:`, `docs:`, or `test:`. Release Please uses commit history to + prepare releases; make the eventual merge or squash subject describe the + change accurately. +5. Open the PR against `main`. Explain the problem, the resulting behavior, + and how you tested it. Call out compatibility and SDK parity implications. + +The build workflow (`.github/workflows/main.yml`) runs on pushes. It tests and +type-checks on Python 3.9–3.13, runs the two lint passes, checks Pyright and its +typing regressions on Python 3.11, and checks corpus freshness against JS +`main`. A local test pass on one interpreter does not cover the full matrix. + +Push follow-up commits rather than force-pushing where possible so review +comments stay attached to the code they discuss. + ## Releasing -1. Bump the version in `setup.cfg` and `setup.py` -2. Merge code to `main` -3. Create a new release on GitHub with your version as the tag (e.g. `v0.2.0`) -4. Run `make dist` to create a distribution file -5. Run `make release` to upload to pypi +Maintainers handle releases through `.github/workflows/release-please.yml`: + +1. Pushes to `main` run Release Please, which prepares a release PR with version + and changelog updates. +2. Merging the release PR lets Release Please create the release and tag. +3. When a release is created, the publishing job runs tests on Python 3.11, + builds the source distribution and wheel with `python -m build`, and uploads + them to PyPI. + +`setup.py` reads the package version from `growthbook/__init__.py`. +`release-please-config.json` and `.release-please-manifest.json` configure and +track releases. Contributors normally do not need to edit version numbers or +`CHANGELOG.md`; include enough detail in the PR for a useful changelog entry. + +`make dist`, `make release`, and the bumpversion files are legacy tooling. +In particular, `make release` uploads directly to PyPI; it is not a local +verification command or the normal release process. + +## Getting help + +- [Slack community](https://slack.growthbook.io?ref=contributing) +- [Open an issue](https://github.com/growthbook/growthbook-python/issues) +- [Python SDK docs](https://docs.growthbook.io/lib/python) + +Found a security vulnerability? Email security@growthbook.io instead of filing +a public issue. See the [security policy](https://github.com/growthbook/growthbook/blob/main/SECURITY.md). + +Contributors are expected to follow the [Code of Conduct](https://github.com/growthbook/growthbook/blob/main/CODE_OF_CONDUCT.md). diff --git a/README.md b/README.md index 48fe16a..c24e352 100644 --- a/README.md +++ b/README.md @@ -394,6 +394,40 @@ gb = GrowthBook( Note: When doing this, you do not need to specify your `api_host` or `client_key` and you don't need to call `gb.load_features()`. +### Saved group references + +Both `GrowthBook` and `GrowthBookClient` support `savedGroupReferencesV2` for +local evaluation. The `savedGroups` payload map can contain legacy ID arrays +or typed entries: + +```json +{ + "savedGroups": { + "beta": {"type": "list", "attributeKey": "id", "values": ["u1", "u2"]}, + "power": {"type": "condition", "condition": {"plan": "pro"}} + } +} +``` + +A rule references either type with a top-level condition such as +`{"$savedGroup": {"id": "beta"}}`. For a list group, an optional `attributeKey` +in the reference overrides the group's attribute. Condition groups evaluate +against the current attributes and can reference other groups; cyclic +references return no match. References compose with `$and`, `$or`, `$nor`, +and `$not`, including in prerequisite conditions (which see the parent's +`value`, not the user's attributes). + +When loading a payload yourself, use `gb.set_payload(payload)` or +`await client.set_payload(payload)` so the saved groups accompany the features. +Legacy `$inGroup` and `$notInGroup` conditions accept both bare arrays and typed +list entries. A missing group still passes `$notInGroup`; a present malformed +entry, condition group, or unknown group type matches neither legacy operator. + +Receiving v2 payloads requires an SDK version registered with the capability +and an SDK Connection set to **All Saved Groups** under **Pass Saved Groups by +reference**. Capability registration follows publication of the supporting SDK +version. This support does not enable v2 for remote-evaluation connections. + ## GrowthBook class The GrowthBook constructor has the following parameters: diff --git a/growthbook/core.py b/growthbook/core.py index d8ddfea..f7ce39a 100644 --- a/growthbook/core.py +++ b/growthbook/core.py @@ -29,38 +29,102 @@ # errored). Matches CONTEXTUAL_BANDIT_FALLBACK_LEAF_ID in the JS SDK. CONTEXTUAL_BANDIT_FALLBACK_LEAF_ID = -1 -def evalCondition(attributes: Dict[str, Any], condition: Dict[str, Any], savedGroups: Optional[Dict[str, Any]] = None) -> bool: +def evalCondition( + attributes: Dict[str, Any], condition: Dict[str, Any], + savedGroups: Optional[Dict[str, Any]] = None, visited: Optional[Set[str]] = None, +) -> bool: + savedGroups = savedGroups if savedGroups is not None else {} + visited = visited if visited is not None else set() for key, value in condition.items(): if key == "$or": - if not evalOr(attributes, value, savedGroups): + if not evalOr(attributes, value, savedGroups, visited): return False elif key == "$nor": - if evalOr(attributes, value, savedGroups): + if evalOr(attributes, value, savedGroups, visited): return False elif key == "$and": - if not evalAnd(attributes, value, savedGroups): + if not evalAnd(attributes, value, savedGroups, visited): return False elif key == "$not": - if evalCondition(attributes, value, savedGroups): + if evalCondition(attributes, value, savedGroups, visited): + return False + elif key == "$savedGroup": + if not _eval_saved_group(attributes, value, savedGroups, visited): return False - elif not evalConditionValue(value, getPath(attributes, key), savedGroups): + elif key == "$savedGroups": + return False + elif not evalConditionValue(value, getPath(attributes, key), savedGroups, visited=visited): return False return True -def evalOr(attributes: Dict[str, Any], conditions: List[Any], savedGroups: Optional[Dict[str, Any]]) -> bool: +def _eval_saved_group( + attributes: Dict[str, Any], reference: Any, saved_groups: Dict[str, Any], visited: Set[str], +) -> bool: + """Resolve a typed saved group, keeping cycle detection local to this branch.""" + if not isinstance(reference, dict): + return False + group_id = reference.get("id") + if not isinstance(group_id, str) or group_id in visited: + return False + if "attributeKey" in reference and not isinstance(reference["attributeKey"], str): + return False + + entry = saved_groups.get(group_id) + if not isinstance(entry, dict): + return False + + try: + if entry.get("type") == "list": + key = reference.get("attributeKey", entry.get("attributeKey")) + values = entry.get("values") + if not isinstance(key, str) or not isinstance(values, list): + return False + return isIn(values, getPath(attributes, key)) + if entry.get("type") == "condition": + condition = entry.get("condition") + if not isinstance(condition, dict): + return False + return evalCondition(attributes, condition, saved_groups, visited | {group_id}) + except (TypeError, ValueError, AttributeError, IndexError, RecursionError): + # Malformed payloads and Python's own stack limit must not escape evaluation. + return False + return False + + +def _saved_group_values(saved_groups: Optional[Dict[str, Any]], group_id: str) -> Optional[List[Any]]: + """Distinguish an absent group (empty list) from a malformed entry (None).""" + if saved_groups is None or group_id not in saved_groups: + return [] + entry = saved_groups[group_id] + if isinstance(entry, list): + return entry + if isinstance(entry, dict) and entry.get("type") == "list": + values = entry.get("values") + if isinstance(values, list): + return values + return None + + +def evalOr( + attributes: Dict[str, Any], conditions: List[Any], savedGroups: Optional[Dict[str, Any]], + visited: Optional[Set[str]] = None, +) -> bool: if len(conditions) == 0: return True for condition in conditions: - if evalCondition(attributes, condition, savedGroups): + if evalCondition(attributes, condition, savedGroups, visited): return True return False -def evalAnd(attributes: Dict[str, Any], conditions: List[Any], savedGroups: Optional[Dict[str, Any]]) -> bool: +def evalAnd( + attributes: Dict[str, Any], conditions: List[Any], savedGroups: Optional[Dict[str, Any]], + visited: Optional[Set[str]] = None, +) -> bool: for condition in conditions: - if not evalCondition(attributes, condition, savedGroups): + if not evalCondition(attributes, condition, savedGroups, visited): return False return True @@ -97,10 +161,13 @@ def getPath(attributes: Dict[str, Any], path: str) -> Any: return None return current -def evalConditionValue(conditionValue: Any, attributeValue: Any, savedGroups: Optional[Dict[str, Any]], insensitive: bool = False) -> bool: +def evalConditionValue( + conditionValue: Any, attributeValue: Any, savedGroups: Optional[Dict[str, Any]], + insensitive: bool = False, visited: Optional[Set[str]] = None, +) -> bool: if isinstance(conditionValue, dict) and isOperatorObject(conditionValue): for key, value in conditionValue.items(): - if not evalOperatorCondition(key, attributeValue, value, savedGroups): + if not evalOperatorCondition(key, attributeValue, value, savedGroups, visited): return False return True @@ -110,16 +177,19 @@ def evalConditionValue(conditionValue: Any, attributeValue: Any, savedGroups: Op return bool(conditionValue == attributeValue) -def elemMatch(condition: Dict[str, Any], attributeValue: Any, savedGroups: Optional[Dict[str, Any]]) -> bool: +def elemMatch( + condition: Dict[str, Any], attributeValue: Any, savedGroups: Optional[Dict[str, Any]], + visited: Optional[Set[str]] = None, +) -> bool: if not isinstance(attributeValue, list): return False for item in attributeValue: if isOperatorObject(condition): - if evalConditionValue(condition, item, savedGroups): + if evalConditionValue(condition, item, savedGroups, visited=visited): return True else: - if evalCondition(item, condition, savedGroups): + if evalCondition(item, condition, savedGroups, visited): return True return False @@ -178,7 +248,10 @@ def _js_strict_equal(a: Any, b: Any) -> bool: return bool(a == b) -def evalOperatorCondition(operator: str, attributeValue: Any, conditionValue: Any, savedGroups: Any) -> bool: +def evalOperatorCondition( + operator: str, attributeValue: Any, conditionValue: Any, savedGroups: Any, + visited: Optional[Set[str]] = None, +) -> bool: if operator == "$eq": return _js_strict_equal(attributeValue, conditionValue) elif operator == "$ne": @@ -215,18 +288,14 @@ def evalOperatorCondition(operator: str, attributeValue: Any, conditionValue: An return paddedVersionString(attributeValue) > paddedVersionString(conditionValue) elif operator == "$vgte": return paddedVersionString(attributeValue) >= paddedVersionString(conditionValue) - elif operator == "$inGroup": + elif operator in ("$inGroup", "$notInGroup"): if not isinstance(conditionValue, str): return False - if not conditionValue in savedGroups: - return False - return isIn(savedGroups[conditionValue] or [], attributeValue) - elif operator == "$notInGroup": - if not isinstance(conditionValue, str): + values = _saved_group_values(savedGroups, conditionValue) + if values is None: return False - if not conditionValue in savedGroups: - return True - return not isIn(savedGroups[conditionValue] or [], attributeValue) + matches = isIn(values, attributeValue) + return matches if operator == "$inGroup" else not matches elif operator == "$regex": try: r = re.compile(conditionValue) @@ -271,19 +340,19 @@ def evalOperatorCondition(operator: str, attributeValue: Any, conditionValue: An return False return not isIn(conditionValue, attributeValue, insensitive=True) elif operator == "$elemMatch": - return elemMatch(conditionValue, attributeValue, savedGroups) + return elemMatch(conditionValue, attributeValue, savedGroups, visited) elif operator == "$size": if not isinstance(attributeValue, list): return False - return evalConditionValue(conditionValue, len(attributeValue), savedGroups) + return evalConditionValue(conditionValue, len(attributeValue), savedGroups, visited=visited) elif operator == "$all": if not isinstance(conditionValue, list): return False - return isInAll(conditionValue, attributeValue, savedGroups, insensitive=False) + return isInAll(conditionValue, attributeValue, savedGroups, insensitive=False, visited=visited) elif operator == "$alli": if not isinstance(conditionValue, list): return False - return isInAll(conditionValue, attributeValue, savedGroups, insensitive=True) + return isInAll(conditionValue, attributeValue, savedGroups, insensitive=True, visited=visited) elif operator == "$exists": if not conditionValue: return attributeValue is None @@ -291,7 +360,7 @@ def evalOperatorCondition(operator: str, attributeValue: Any, conditionValue: An elif operator == "$type": return bool(getType(attributeValue) == conditionValue) elif operator == "$not": - return not evalConditionValue(conditionValue, attributeValue, savedGroups) + return not evalConditionValue(conditionValue, attributeValue, savedGroups, visited=visited) return False def paddedVersionString(input: Any) -> str: @@ -347,10 +416,17 @@ def case_fold(val: Any) -> Any: # Case-sensitive behavior (original) if isinstance(attributeValue, list): - return bool(set(conditionValue) & set(attributeValue)) + try: + return bool(set(conditionValue) & set(attributeValue)) + except TypeError: + # JSON arrays may contain objects or arrays, which cannot be hashed. + return any(value in conditionValue for value in attributeValue) return attributeValue in conditionValue -def isInAll(conditionValue: List[Any], attributeValue: Any, savedGroups: Optional[Dict[str, Any]], insensitive: bool = False) -> bool: +def isInAll( + conditionValue: List[Any], attributeValue: Any, savedGroups: Optional[Dict[str, Any]], + insensitive: bool = False, visited: Optional[Set[str]] = None, +) -> bool: """Check if attributeValue (array) contains all elements in conditionValue""" if not isinstance(attributeValue, list): return False @@ -358,7 +434,7 @@ def isInAll(conditionValue: List[Any], attributeValue: Any, savedGroups: Optiona for cond in conditionValue: passing = False for attr in attributeValue: - if evalConditionValue(cond, attr, savedGroups, insensitive): + if evalConditionValue(cond, attr, savedGroups, insensitive, visited): passing = True break if not passing: diff --git a/tests/cases.json b/tests/cases.json index 5d4e57f..f8ac7f4 100644 --- a/tests/cases.json +++ b/tests/cases.json @@ -1,5 +1,5 @@ { - "specVersion": "0.8.0", + "specVersion": "0.9.0", "evalCondition": [ [ "$not - pass", @@ -4160,6 +4160,300 @@ ] }, true + ], + [ + "$vgt - date-based version", + { + "version": { + "$vgt": "20260909.9.9" + } + }, + { + "version": "20260910.1.1" + }, + true + ], + [ + "$vlt - oversized condition operand", + { + "version": { + "$vlt": "123456789" + } + }, + { + "version": "1.0.0" + }, + true + ], + [ + "$veq - oversized on both sides", + { + "version": { + "$veq": "123456789" + } + }, + { + "version": "123456789" + }, + true + ], + [ + "$vgt - oversized pre-release segment", + { + "version": { + "$vgt": "1.2.3-rc.123455" + } + }, + { + "version": "1.2.3-rc.123456" + }, + true + ], + [ + "$vlte - oversized equal segments", + { + "version": { + "$vlte": "20260910.1.0" + } + }, + { + "version": "20260910.1.0" + }, + true + ], + [ + "$vlt - absent attribute is below any release", + { + "version": { + "$vlt": "0.0.1" + } + }, + {}, + true + ], + [ + "$vgt - absent attribute is not above a release", + { + "version": { + "$vgt": "0.0.1" + } + }, + {}, + false + ], + [ + "$veq - absent attribute equals 0", + { + "version": { + "$veq": "0" + } + }, + {}, + true + ], + [ + "$vne - absent attribute differs from a real version", + { + "version": { + "$vne": "1.2.3" + } + }, + {}, + true + ], + [ + "$vgt - numeric attribute is stringified", + { + "version": { + "$vgt": "1.0.0" + } + }, + { + "version": 2 + }, + true + ], + [ + "$veq - numeric attribute equals numeric condition", + { + "version": { + "$veq": 2 + } + }, + { + "version": 2 + }, + true + ], + [ + "$veq - float attribute splits on the decimal point", + { + "version": { + "$veq": "1.5" + } + }, + { + "version": 1.5 + }, + true + ], + [ + "$veq - empty string attribute is version 0", + { + "version": { + "$veq": "0" + } + }, + { + "version": "" + }, + true + ], + [ + "$veq - boolean attribute is version 0", + { + "version": { + "$veq": "0" + } + }, + { + "version": true + }, + true + ], + [ + "$veq - array attribute is version 0", + { + "version": { + "$veq": "0" + } + }, + { + "version": [ + "1.0.0" + ] + }, + true + ], + [ + "$veq - null attribute is version 0", + { + "version": { + "$veq": "0" + } + }, + { + "version": null + }, + true + ], + [ + "$veq - leading v is ignored", + { + "version": { + "$veq": "1.2.3" + } + }, + { + "version": "v1.2.3" + }, + true + ], + [ + "$veq - v inside a pre-release tag is kept", + { + "version": { + "$veq": "1.2.3-dev" + } + }, + { + "version": "1.2.3-dev" + }, + true + ], + [ + "$vne - 1.2.3-dev is not 1.2.3-de", + { + "version": { + "$vne": "1.2.3-de" + } + }, + { + "version": "1.2.3-dev" + }, + true + ], + [ + "$veq - v inside a dotted pre-release is kept", + { + "version": { + "$veq": "1.0.0-alpha.v1" + } + }, + { + "version": "1.0.0-alpha.v1" + }, + true + ], + [ + "$veq - empty middle segment", + { + "version": { + "$veq": "1..2" + } + }, + { + "version": "1..2" + }, + true + ], + [ + "$vne - empty segment changes the part count", + { + "version": { + "$vne": "1.2" + } + }, + { + "version": "1..2" + }, + true + ], + [ + "$veq - leading separator", + { + "version": { + "$veq": "-1.2.3" + } + }, + { + "version": "-1.2.3" + }, + true + ], + [ + "$vne - non-ASCII digit is not the ASCII digit", + { + "version": { + "$vne": "1.2.3-3" + } + }, + { + "version": "1.2.3-\u0663" + }, + true + ], + [ + "$veq - non-ASCII digits compare as a tag", + { + "version": { + "$veq": "\u0661.\u0662.\u0663" + } + }, + { + "version": "\u0661.\u0662.\u0663" + }, + true ] ], "hash": [ @@ -14094,5 +14388,1467 @@ "ramped" ] ] - ] + ], + "savedGroupReferencesV2": { + "evalCondition": [ + [ + "list entry matches when the attribute is in the list", + { + "$savedGroup": { "id": "grp_list" } + }, + { + "id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "list entry does not match when the attribute is absent from the list", + { + "$savedGroup": { "id": "grp_list" } + }, + { + "id": "u_9" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "condition entry matches when its condition passes", + { + "$savedGroup": { "id": "grp_cond" } + }, + { + "plan": "pro" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "condition entry does not match when its condition fails", + { + "$savedGroup": { "id": "grp_cond" } + }, + { + "plan": "free" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "follows a chain of condition groups", + { + "$savedGroup": { "id": "grp_a" } + }, + { + "country": "US" + }, + true, + { + "grp_a": { + "type": "condition", + "condition": { + "$savedGroup": { "id": "grp_b" } + } + }, + "grp_b": { + "type": "condition", + "condition": { + "$savedGroup": { "id": "grp_c" } + } + }, + "grp_c": { + "type": "condition", + "condition": { + "country": "US" + } + } + } + ], + [ + "follows a chain and fails at the end", + { + "$savedGroup": { "id": "grp_a" } + }, + { + "country": "CA" + }, + false, + { + "grp_a": { + "type": "condition", + "condition": { + "$savedGroup": { "id": "grp_b" } + } + }, + "grp_b": { + "type": "condition", + "condition": { + "$savedGroup": { "id": "grp_c" } + } + }, + "grp_c": { + "type": "condition", + "condition": { + "country": "US" + } + } + } + ], + [ + "resolves inside $not", + { + "$not": { + "$savedGroup": { "id": "grp_list" } + } + }, + { + "id": "u_9" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "resolves inside $and", + { + "$and": [ + { + "$savedGroup": { "id": "grp_list" } + }, + { + "$savedGroup": { "id": "grp_cond" } + } + ] + }, + { + "id": "u_1", + "plan": "pro" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "resolves inside $or", + { + "$or": [ + { + "$savedGroup": { "id": "grp_list" } + }, + { + "$savedGroup": { "id": "grp_cond" } + } + ] + }, + { + "id": "u_9", + "plan": "pro" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "match all: passes only when in every group", + { + "$and": [ + { + "$savedGroup": { "id": "grp_list" } + }, + { + "$savedGroup": { "id": "grp_cond" } + } + ] + }, + { + "id": "u_1", + "plan": "free" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "match any: passes when in one group", + { + "$or": [ + { + "$savedGroup": { "id": "grp_list" } + }, + { + "$savedGroup": { "id": "grp_cond" } + } + ] + }, + { + "id": "u_9", + "plan": "free" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "match none: an AND of NOTs passes when in neither group", + { + "$and": [ + { + "$not": { + "$savedGroup": { "id": "grp_list" } + } + }, + { + "$not": { + "$savedGroup": { "id": "grp_cond" } + } + } + ] + }, + { + "id": "u_9", + "plan": "free" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "match none: an AND of NOTs fails when in one group", + { + "$and": [ + { + "$not": { + "$savedGroup": { "id": "grp_list" } + } + }, + { + "$not": { + "$savedGroup": { "id": "grp_cond" } + } + } + ] + }, + { + "id": "u_1", + "plan": "free" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "a negated pair passes when in only one group, where an AND of NOTs fails", + { + "$not": { + "$and": [ + { + "$savedGroup": { "id": "grp_list" } + }, + { + "$savedGroup": { "id": "grp_cond" } + } + ] + } + }, + { + "id": "u_1", + "plan": "free" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "combines with plain attribute targeting", + { + "$and": [ + { + "country": "US" + }, + { + "$savedGroup": { "id": "grp_list" } + } + ] + }, + { + "country": "US", + "id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "combines with plain attribute targeting and fails on the attribute", + { + "$and": [ + { + "country": "US" + }, + { + "$savedGroup": { "id": "grp_list" } + } + ] + }, + { + "country": "CA", + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "list entry intersects when the attribute is an array", + { + "$savedGroup": { "id": "grp_list" } + }, + { + "id": ["u_9", "u_2"] + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "list entry matches numeric values", + { + "$savedGroup": { "id": "grp_num" } + }, + { + "age": 21 + }, + true, + { + "grp_num": { + "type": "list", + "attributeKey": "age", + "values": [21, 30] + } + } + ], + [ + "a cycle resolves to false instead of hanging", + { + "$savedGroup": { "id": "grp_x" } + }, + { + "id": "u_1" + }, + false, + { + "grp_x": { + "type": "condition", + "condition": { + "$savedGroup": { "id": "grp_y" } + } + }, + "grp_y": { + "type": "condition", + "condition": { + "$savedGroup": { "id": "grp_x" } + } + } + } + ], + [ + "an unknown group type matches nobody", + { + "$savedGroup": { "id": "grp_future" } + }, + { + "id": "u_1" + }, + false, + { + "grp_future": { + "type": "somethingNew", + "values": ["u_1"] + } + } + ], + [ + "an id that is not in the payload matches nobody", + { + "$savedGroup": { "id": "grp_missing" } + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "a non-string operator value matches nobody", + { + "$savedGroup": ["grp_list"] + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "a list entry with no values matches nobody", + { + "$savedGroup": { "id": "grp_bad" } + }, + { + "id": "u_1" + }, + false, + { + "grp_bad": { + "type": "list", + "attributeKey": "id" + } + } + ], + [ + "a condition entry with no condition matches nobody", + { + "$savedGroup": { "id": "grp_bad" } + }, + { + "id": "u_1" + }, + false, + { + "grp_bad": { + "type": "condition" + } + } + ], + [ + "$savedGroup cannot resolve a legacy bare array", + { + "$savedGroup": { "id": "grp_legacy" } + }, + { + "id": "u_1" + }, + false, + { + "grp_legacy": ["u_1"], + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "$inGroup still works beside typed entries", + { + "id": { + "$inGroup": "grp_legacy" + } + }, + { + "id": "u_1" + }, + true, + { + "grp_legacy": ["u_1"], + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "$inGroup resolves a typed list entry", + { + "id": { + "$inGroup": "grp_list" + } + }, + { + "id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "$inGroup uses the attribute it sits on, not the one in the entry", + { + "backup_id": { + "$inGroup": "grp_list" + } + }, + { + "id": "u_9", + "backup_id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "$inGroup matches nobody for a typed condition entry", + { + "id": { + "$inGroup": "grp_cond" + } + }, + { + "id": "u_1", + "plan": "pro" + }, + false, + { + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "$notInGroup resolves a typed list entry and fails for a member", + { + "id": { + "$notInGroup": "grp_list" + } + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "$notInGroup resolves a typed list entry and passes for a non-member", + { + "id": { + "$notInGroup": "grp_list" + } + }, + { + "id": "u_9" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "$notInGroup fails closed for a typed condition entry rather than passing everyone", + { + "id": { + "$notInGroup": "grp_cond" + } + }, + { + "id": "u_1", + "plan": "free" + }, + false, + { + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "$notInGroup fails closed for a group type added after the SDK was built", + { + "id": { + "$notInGroup": "grp_future" + } + }, + { + "id": "u_1" + }, + false, + { + "grp_future": { + "type": "somethingNew", + "values": ["u_1"] + } + } + ], + [ + "$notInGroup fails closed for a malformed entry rather than passing everyone", + { + "id": { + "$notInGroup": "grp_bad" + } + }, + { + "id": "u_1" + }, + false, + { + "grp_bad": null + } + ], + [ + "$inGroup matches nobody for a malformed entry", + { + "id": { + "$inGroup": "grp_bad" + } + }, + { + "id": "u_1" + }, + false, + { + "grp_bad": null + } + ], + [ + "$notInGroup still passes for an id that is not in the payload", + { + "id": { + "$notInGroup": "grp_missing" + } + }, + { + "id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "the authoring operator $savedGroups is not understood", + { + "$savedGroups": ["grp_list"] + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "an attributeKey override checks that attribute instead of the entry's", + { + "$savedGroup": { + "id": "grp_list", + "attributeKey": "backup_id" + } + }, + { + "id": "u_9", + "backup_id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "an attributeKey override does not fall back to the entry's attribute", + { + "$savedGroup": { + "id": "grp_list", + "attributeKey": "backup_id" + } + }, + { + "id": "u_1", + "backup_id": "u_9" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "an attributeKey override resolves inside $not", + { + "$not": { + "$savedGroup": { + "id": "grp_list", + "attributeKey": "backup_id" + } + } + }, + { + "id": "u_1", + "backup_id": "u_9" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "a non-string attributeKey matches nobody rather than using the entry's", + { + "$savedGroup": { + "id": "grp_list", + "attributeKey": 42 + } + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "an attributeKey override is ignored for a condition entry", + { + "$savedGroup": { + "id": "grp_cond", + "attributeKey": "backup_id" + } + }, + { + "plan": "pro" + }, + true, + { + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + ], + [ + "an unknown key inside the reference is ignored", + { + "$savedGroup": { + "id": "grp_list", + "somethingNew": "ignore me" + } + }, + { + "id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "a bare string id is not a reference and matches nobody", + { + "$savedGroup": "grp_list" + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "a reference with no id matches nobody", + { + "$savedGroup": { + "attributeKey": "id" + } + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "an error marker matches nobody", + { + "__sgInvalid__": "grp_cond" + }, + { + "id": "u_1" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "a negated error marker passes everyone, as an unresolved exclusion did before", + { + "$not": { + "__sgUnknown__": "grp_missing" + } + }, + { + "id": "u_1" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "an error marker fails the $and it sits in", + { + "$and": [ + { + "country": "US" + }, + { + "__sgInvalid__": "grp_cond" + } + ] + }, + { + "id": "u_1", + "country": "US" + }, + false, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ], + [ + "an error marker does not fail an $or whose other branch passes", + { + "$or": [ + { + "country": "US" + }, + { + "__sgInvalid__": "grp_cond" + } + ] + }, + { + "id": "u_1", + "country": "US" + }, + true, + { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + } + } + ] + ], + "feature": [ + [ + "a rule condition using a saved group reference", + { + "attributes": { + "id": "u_1", + "plan": "pro" + }, + "savedGroups": { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + }, + "features": { + "flag": { + "defaultValue": false, + "rules": [ + { + "condition": { + "$savedGroup": { "id": "grp_cond" } + }, + "force": true, + "id": "rule_1" + } + ] + } + } + }, + "flag", + { + "value": true, + "on": true, + "off": false, + "source": "force", + "ruleId": "rule_1" + } + ], + [ + "a rule condition using a saved group reference that does not match", + { + "attributes": { + "id": "u_1", + "plan": "free" + }, + "savedGroups": { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + }, + "features": { + "flag": { + "defaultValue": false, + "rules": [ + { + "condition": { + "$savedGroup": { "id": "grp_cond" } + }, + "force": true, + "id": "rule_1" + } + ] + } + } + }, + "flag", + { + "value": false, + "on": false, + "off": true, + "source": "defaultValue", + "ruleId": "" + } + ], + [ + "a prerequisite gate that itself uses a saved group reference", + { + "attributes": { + "id": "u_1" + }, + "savedGroups": { + "grp_parent_on": { + "type": "condition", + "condition": { + "value": true + } + } + }, + "features": { + "parent": { + "defaultValue": true + }, + "child": { + "defaultValue": false, + "rules": [ + { + "parentConditions": [ + { + "id": "parent", + "condition": { + "$savedGroup": { "id": "grp_parent_on" } + } + } + ], + "force": true, + "id": "child_rule" + } + ] + } + } + }, + "child", + { + "value": true, + "on": true, + "off": false, + "source": "force", + "ruleId": "child_rule" + } + ], + [ + "a prerequisite gate whose saved group reference does not match", + { + "attributes": { + "id": "u_1" + }, + "savedGroups": { + "grp_parent_on": { + "type": "condition", + "condition": { + "value": true + } + } + }, + "features": { + "parent": { + "defaultValue": false + }, + "child": { + "defaultValue": false, + "rules": [ + { + "parentConditions": [ + { + "id": "parent", + "condition": { + "$savedGroup": { "id": "grp_parent_on" } + } + } + ], + "force": true, + "id": "child_rule" + } + ] + } + } + }, + "child", + { + "value": false, + "on": false, + "off": true, + "source": "defaultValue", + "ruleId": "" + } + ] + ], + "run": [ + [ + "an experiment targeted by a saved group reference", + { + "attributes": { + "id": "u_1", + "plan": "pro" + }, + "savedGroups": { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + }, + { + "key": "sg-exp", + "variations": [0, 1], + "condition": { + "$savedGroup": { "id": "grp_cond" } + } + }, + 0, + true, + true + ], + [ + "an experiment skipped because the saved group does not match", + { + "attributes": { + "id": "u_1", + "plan": "free" + }, + "savedGroups": { + "grp_list": { + "type": "list", + "attributeKey": "id", + "values": ["u_1", "u_2"] + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + } + }, + { + "key": "sg-exp", + "variations": [0, 1], + "condition": { + "$savedGroup": { "id": "grp_cond" } + } + }, + 0, + false, + false + ], + [ + "an experiment gated by a prerequisite that uses a saved group reference", + { + "attributes": { + "id": "u_1", + "plan": "pro" + }, + "savedGroups": { + "grp_parent_on": { + "type": "condition", + "condition": { + "value": true + } + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + }, + "features": { + "parent": { + "defaultValue": true + } + } + }, + { + "key": "sg-prereq-exp", + "variations": [0, 1], + "parentConditions": [ + { + "id": "parent", + "condition": { + "$savedGroup": { "id": "grp_parent_on" } + } + } + ] + }, + 0, + true, + true + ], + [ + "an experiment blocked because its prerequisite saved group does not match", + { + "attributes": { + "id": "u_1", + "plan": "pro" + }, + "savedGroups": { + "grp_parent_on": { + "type": "condition", + "condition": { + "value": true + } + }, + "grp_cond": { + "type": "condition", + "condition": { + "plan": "pro" + } + } + }, + "features": { + "parent": { + "defaultValue": false + } + } + }, + { + "key": "sg-prereq-exp", + "variations": [0, 1], + "parentConditions": [ + { + "id": "parent", + "condition": { + "$savedGroup": { "id": "grp_parent_on" } + } + } + ] + }, + 0, + false, + false + ] + ] + } } diff --git a/tests/scripts/check_corpus_freshness.py b/tests/scripts/check_corpus_freshness.py index ca6838f..75146f2 100644 --- a/tests/scripts/check_corpus_freshness.py +++ b/tests/scripts/check_corpus_freshness.py @@ -55,7 +55,7 @@ "packages/sdk-js/test/cases.json" ) -# Top-level keys to diff. Other keys in cases.json (specVersion, decrypt +# Suite paths to diff. Other keys in cases.json (specVersion, decrypt # binary blobs, urlRedirect which Python doesn't yet wire) are skipped # either because they're scalar metadata or because the divergence is # tracked separately. @@ -71,6 +71,9 @@ "inNamespace", "getEqualWeights", "stickyBucket", + "savedGroupReferencesV2.evalCondition", + "savedGroupReferencesV2.feature", + "savedGroupReferencesV2.run", # Python-local extension today (reported as extras); once the JS SDK # adopts the section, drift checking picks it up automatically. "trackingCalls", @@ -191,10 +194,8 @@ def _diff( drift_skip = skip.get("drift", {}) for key in KEYS_TO_DIFF: - js_list = js_cases.get(key, []) - py_list = py_cases.get(key, []) - if not isinstance(js_list, list) or not isinstance(py_list, list): - continue + js_list = _section_cases(js_cases, key) + py_list = _section_cases(py_cases, key) js_grouped = _case_signatures_grouped(js_list) py_grouped = _case_signatures_grouped(py_list) @@ -238,6 +239,16 @@ def _diff( return actionable_missing, skipped_missing, extras, actionable_drift, skipped_drift +def _section_cases(corpus: dict, key: str) -> list: + """Read a suite, including capability suites nested under a top-level key.""" + section = corpus + for part in key.split("."): + if not isinstance(section, dict): + return [] + section = section.get(part, []) + return section if isinstance(section, list) else [] + + def _spec_versions(js_cases: dict, py_cases: dict) -> Tuple[str, str]: return ( str(js_cases.get("specVersion", "")), diff --git a/tests/test_corpus_freshness.py b/tests/test_corpus_freshness.py new file mode 100644 index 0000000..cc2ba3b --- /dev/null +++ b/tests/test_corpus_freshness.py @@ -0,0 +1,37 @@ +"""Capability sub-suites must participate in the corpus drift check.""" + +import runpy +from pathlib import Path + +import pytest + + +diff_corpora = runpy.run_path( + str(Path(__file__).parent / "scripts" / "check_corpus_freshness.py") +)["_diff"] + + +@pytest.mark.parametrize("suite", ["evalCondition", "feature", "run"]) +def test_nested_suites_report_missing_drift_and_local_cases(suite): + upstream = {"savedGroupReferencesV2": {suite: [["missing", True], ["changed", True]]}} + local = {"savedGroupReferencesV2": {suite: [["changed", False], ["local", True]]}} + key = "savedGroupReferencesV2." + suite + missing, _, extras, drift, _ = diff_corpora(upstream, local, {}) + assert missing[key] == ["missing"] + assert drift[key] == ["changed"] + assert extras[key] == ["local"] + + missing, skipped, _, drift, skipped_drift = diff_corpora(upstream, local, { + "missing": {key: {"missing"}}, "drift": {key: {"changed"}}, + }) + assert missing[key] == drift[key] == [] + assert skipped[key] == ["missing"] + assert skipped_drift[key] == ["changed"] + + +@pytest.mark.parametrize("capability", [None, {}, [], {"evalCondition": {}}]) +def test_missing_or_malformed_capability_suite_cannot_hide_missing_cases(capability): + upstream = {"savedGroupReferencesV2": {"evalCondition": [["new", True]]}} + local = {"savedGroupReferencesV2": capability} + missing, _, _, _, _ = diff_corpora(upstream, local, {}) + assert missing["savedGroupReferencesV2.evalCondition"] == ["new"] diff --git a/tests/test_saved_groups.py b/tests/test_saved_groups.py new file mode 100644 index 0000000..728b3b3 --- /dev/null +++ b/tests/test_saved_groups.py @@ -0,0 +1,245 @@ +"""Saved group references v2 conformance and Python-specific regressions.""" + +import json +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + +from growthbook import Experiment, GrowthBook, GrowthBookClient, Options, UserContext +from growthbook.core import evalCondition + + +CASES = json.loads(Path(__file__).with_name("cases.json").read_text())[ + "savedGroupReferencesV2" +] + + +@pytest.mark.parametrize("case", CASES["evalCondition"], ids=lambda case: case[0]) +def test_condition_conformance(case): + _, condition, attributes, expected, saved_groups = case + assert evalCondition(attributes, condition, saved_groups) is expected + + +@pytest.mark.parametrize("case", CASES["feature"], ids=lambda case: case[0]) +def test_sync_feature_conformance(case): + _, context, key, expected = deepcopy(case) + client = GrowthBook(**context) + try: + assert client.eval_feature(key).to_dict() == expected + finally: + client.destroy() + + +@pytest.mark.parametrize("case", CASES["run"], ids=lambda case: case[0]) +def test_sync_experiment_conformance(case): + _, context, experiment, value, in_experiment, hash_used = deepcopy(case) + client = GrowthBook(**context) + try: + result = client.run(Experiment(**experiment)) + assert (result.value, result.inExperiment, result.hashUsed) == ( + value, in_experiment, hash_used + ) + finally: + client.destroy() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", CASES["feature"], ids=lambda case: case[0]) +async def test_async_feature_conformance(case): + _, context, key, expected = deepcopy(case) + client = GrowthBookClient(Options()) + try: + await client.set_payload(context) + result = await client.eval_feature(key, UserContext(attributes=context["attributes"])) + assert result.to_dict() == expected + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", CASES["run"], ids=lambda case: case[0]) +async def test_async_experiment_conformance(case): + _, context, experiment, value, in_experiment, hash_used = deepcopy(case) + client = GrowthBookClient(Options()) + try: + await client.set_payload(context) + result = await client.run(Experiment(**experiment), UserContext(attributes=context["attributes"])) + assert (result.value, result.inExperiment, result.hashUsed) == ( + value, in_experiment, hash_used + ) + finally: + await client.close() + + +@pytest.mark.parametrize("entry", [ + {"type": "list", "attributeKey": "id", "values": ["u1"]}, + {"type": "condition", "condition": {"id": "u1"}}, +]) +@pytest.mark.parametrize("reference", [ + None, False, 7, "g", [], ["g"], {}, + {"id": None}, {"id": 7}, {"id": []}, {"id": {}}, + *({"id": "g", "attributeKey": key} for key in [None, 7, False, [], {}]), +]) +def test_invalid_references_fail_closed(reference, entry): + assert evalCondition({"id": "u1"}, {"$savedGroup": reference}, {"g": entry}) is False + + +@pytest.mark.parametrize("entry", [ + None, False, 7, "g", {}, {"type": "future"}, {"type": []}, + {"type": "list", "attributeKey": "id"}, + *({"type": "list", "attributeKey": "id", "values": values} for values in [None, 7, {}, "u1"]), + {"type": "condition"}, + *({"type": "condition", "condition": condition} for condition in [ + None, 7, [], "g", {"$and": 7}, {"$not": None}, {"id": {"": True}}, + ]), +]) +def test_malformed_entries_fail_closed(entry): + groups = {"bad": entry, "good": {"type": "condition", "condition": {}}} + attributes = {"id": "u1"} + for condition in [ + {"$savedGroup": {"id": "bad"}}, + {"id": {"$inGroup": "bad"}}, + {"id": {"$notInGroup": "bad"}}, + ]: + assert evalCondition(attributes, condition, groups) is False + assert evalCondition(attributes, {"$savedGroup": {"id": "good"}}, groups) is True + assert evalCondition(attributes, {"id": {"$notInGroup": "missing"}}, groups) is True + + +@pytest.mark.parametrize("groups", [None, {}]) +def test_legacy_operators_without_saved_groups(groups): + assert evalCondition({"id": "u1"}, {"id": {"$inGroup": "missing"}}, groups) is False + assert evalCondition({"id": "u1"}, {"id": {"$notInGroup": "missing"}}, groups) is True + + +@pytest.mark.parametrize("entry_key", [None, 7, "id"]) +def test_override_and_legacy_operators_supply_the_attribute(entry_key): + entry = {"type": "list", "attributeKey": entry_key, "values": ["u1"]} + groups = {"g": entry} + attributes = {"profile": {"id": "u1"}, "id": "u2"} + assert evalCondition(attributes, {"$savedGroup": {"id": "g", "attributeKey": "profile.id"}}, groups) + assert evalCondition(attributes, {"profile.id": {"$inGroup": "g"}}, groups) + assert not evalCondition(attributes, {"$savedGroup": {"id": "g"}}, groups) + + +def test_overrides_and_visited_groups_do_not_leak_to_siblings(): + groups = { + "a": {"type": "condition", "condition": {"$savedGroup": {"id": "b"}}}, + "b": {"type": "list", "attributeKey": "id", "values": ["u1"]}, + } + original = deepcopy(groups) + attributes = {"id": "u1", "backup": "u2"} + reference = {"$savedGroup": {"id": "a", "future": {"key": True}}} + assert evalCondition(attributes, {"$and": [reference, reference]}, groups) + assert evalCondition(attributes, {"$or": [{"$and": [reference, {"missing": True}]}, reference]}, groups) + assert evalCondition(attributes, {"$and": [ + {"$not": {"$savedGroup": {"id": "b", "attributeKey": "backup"}}}, + {"$savedGroup": {"id": "a", "attributeKey": "backup"}}, + ]}, groups) + assert groups == original + + +@pytest.mark.parametrize("condition, expected", [ + ({"$savedGroup": {"id": "g"}}, False), + ({"$and": [{"$savedGroup": {"id": "g"}}]}, False), + ({"$or": [{"$savedGroup": {"id": "g"}}]}, False), + ({"$nor": [{"$savedGroup": {"id": "g"}}]}, True), + ({"$not": {"$savedGroup": {"id": "g"}}}, True), + ({"$not": {"$not": {"$savedGroup": {"id": "g"}}}}, False), +]) +def test_cycles_preserve_boolean_composition(condition, expected): + groups = {"g": {"type": "condition", "condition": condition}} + assert evalCondition({}, {"$savedGroup": {"id": "g"}}, groups) is expected + + +@pytest.mark.parametrize("operator", ["$elemMatch", "$all", "$alli", "$not"]) +def test_visited_groups_survive_nested_value_evaluation(operator): + nested = {"$elemMatch": {"marker": True, "$savedGroup": {"id": "g"}}} + items = [{"marker": True, "terminal": True}] + if operator in ("$all", "$alli"): + nested = {operator: [nested]} + items = [items] + elif operator == "$not": + nested = {"$not": {"$not": nested}} + groups = {"g": {"type": "condition", "condition": {"$or": [ + {"terminal": True}, {"items": nested}, + ]}}} + # Losing the visited set would let the nested element's terminal branch pass. + assert not evalCondition({"items": items}, {"$savedGroup": {"id": "g"}}, groups) + + +def test_long_acyclic_chain_has_no_sdk_depth_limit(): + groups = {str(i): {"type": "condition", "condition": {"$savedGroup": {"id": str(i + 1)}}} + for i in range(150)} + groups["150"] = {"type": "condition", "condition": {"id": "u1"}} + assert evalCondition({"id": "u1"}, {"$savedGroup": {"id": "0"}}, groups) + + +def test_cycle_longer_than_python_stack_fails_closed(): + count = sys.getrecursionlimit() + groups = {str(i): {"type": "condition", "condition": {"$savedGroup": {"id": str((i + 1) % count)}}} + for i in range(count)} + assert not evalCondition({}, {"$savedGroup": {"id": "0"}}, groups) + + +@pytest.mark.parametrize("marker", ["__sgInvalid__", "__sgUnknown__", "__sgCycle__", "__sgMaxDepth__"]) +def test_error_markers_keep_boolean_semantics(marker): + condition = {marker: "g"} + assert not evalCondition({}, condition) + assert evalCondition({}, {"$not": condition}) + assert not evalCondition({}, {"$and": [{}, condition]}) + assert evalCondition({}, {"$or": [condition, {}]}) + + +def test_saved_group_is_only_a_top_level_operator(): + groups = {"g": {"type": "condition", "condition": {}}} + assert not evalCondition({"id": "u1"}, {"id": {"$savedGroup": {"id": "g"}}}, groups) + assert not evalCondition({}, {"$savedGroups": None}, groups) + + +def test_array_membership_handles_unhashable_attributes(): + attributes = {"id": [{"nested": True}, ["u2"], "u1"]} + groups = {"g": {"type": "list", "attributeKey": "id", "values": ["u1"]}} + assert evalCondition(attributes, {"id": {"$in": ["u1"]}}) + assert evalCondition(attributes, {"id": {"$inGroup": "g"}}, groups) + assert not evalCondition(attributes, {"id": {"$notInGroup": "g"}}, groups) + assert evalCondition(attributes, {"$savedGroup": {"id": "g"}}, groups) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("attributes, expected", [ + ({"id": "u1", "plan": "pro"}, True), + ({"id": "u2", "plan": "pro"}, False), + ({"id": "u1", "plan": "free"}, False), +]) +async def test_switching_payload_formats_preserves_results(attributes, expected): + legacy = { + "savedGroups": {"beta": ["u1"]}, + "features": {"flag": {"defaultValue": False, "rules": [{ + "condition": {"id": {"$inGroup": "beta"}, "plan": "pro"}, "force": True, + }]}}, + } + typed = { + "savedGroups": { + "beta": {"type": "list", "attributeKey": "id", "values": ["u1"]}, + "power": {"type": "condition", "condition": {"plan": "pro"}}, + }, + "features": {"flag": {"defaultValue": False, "rules": [{ + "condition": {"$and": [{"$savedGroup": {"id": "beta"}}, {"$savedGroup": {"id": "power"}}]}, + "force": True, + }]}}, + } + sync_client = GrowthBook(attributes=attributes) + async_client = GrowthBookClient(Options()) + user = UserContext(attributes=attributes) + try: + for payload in [legacy, typed, legacy]: + sync_client.set_payload(deepcopy(payload)) + await async_client.set_payload(deepcopy(payload)) + assert sync_client.is_on("flag") is expected + assert await async_client.is_on("flag", user) is expected + finally: + sync_client.destroy() + await async_client.close() From 75c1c1a1789dd53f55cdecf6dac23084ba8058f8 Mon Sep 17 00:00:00 2001 From: Andy White Date: Fri, 25 Sep 2026 16:43:32 -0600 Subject: [PATCH 2/6] perf: avoid saved group overhead in legacy evaluation --- CONTRIBUTING.md | 6 +- README.md | 7 +- growthbook/core.py | 17 +-- tests/scripts/README.md | 14 +++ tests/scripts/benchmark_saved_groups.py | 132 ++++++++++++++++++++++++ tests/test_saved_groups.py | 103 +++++++++++++++++- 6 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 tests/scripts/benchmark_saved_groups.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c31fe8..7f67851 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -183,11 +183,15 @@ Run benchmarks from the repository root with the virtual environment active: ```bash python tests/scripts/benchmark_eval_overhead.py 100000 python tests/scripts/benchmark_async_client.py +python tests/scripts/benchmark_saved_groups.py ``` The first measures sync evaluation overhead; the second measures async throughput, latency, and event-loop lag with simulated sticky bucket services. -Neither needs external services. Include before/after results in the PR when +The third measures ordinary and saved-group conditions through both clients; +use `--sdk-path /path/to/checkout --legacy-only` to compare existing workloads +before and after a change. None needs external services. Include before/after +results in the PR when changing evaluation performance or concurrency behavior, using the same Python version, machine, and workload for both runs. diff --git a/README.md b/README.md index c24e352..43bebf1 100644 --- a/README.md +++ b/README.md @@ -425,8 +425,11 @@ entry, condition group, or unknown group type matches neither legacy operator. Receiving v2 payloads requires an SDK version registered with the capability and an SDK Connection set to **All Saved Groups** under **Pass Saved Groups by -reference**. Capability registration follows publication of the supporting SDK -version. This support does not enable v2 for remote-evaluation connections. +reference**. Before publication, a new version can be registered with +`prerelease: true` for testing by entering that version manually on the connection. +Remove the prerelease flag and regenerate `CAPABILITIES.md` when the version is +published. Remote-evaluation connections require the proxy's JS SDK dependency +to support v2 before the back end enables the capability for them. ## GrowthBook class diff --git a/growthbook/core.py b/growthbook/core.py index f7ce39a..26ead4e 100644 --- a/growthbook/core.py +++ b/growthbook/core.py @@ -33,8 +33,6 @@ def evalCondition( attributes: Dict[str, Any], condition: Dict[str, Any], savedGroups: Optional[Dict[str, Any]] = None, visited: Optional[Set[str]] = None, ) -> bool: - savedGroups = savedGroups if savedGroups is not None else {} - visited = visited if visited is not None else set() for key, value in condition.items(): if key == "$or": if not evalOr(attributes, value, savedGroups, visited): @@ -59,18 +57,19 @@ def evalCondition( return True def _eval_saved_group( - attributes: Dict[str, Any], reference: Any, saved_groups: Dict[str, Any], visited: Set[str], + attributes: Dict[str, Any], reference: Any, saved_groups: Optional[Dict[str, Any]], + visited: Optional[Set[str]], ) -> bool: """Resolve a typed saved group, keeping cycle detection local to this branch.""" if not isinstance(reference, dict): return False group_id = reference.get("id") - if not isinstance(group_id, str) or group_id in visited: + if not isinstance(group_id, str) or (visited is not None and group_id in visited): return False if "attributeKey" in reference and not isinstance(reference["attributeKey"], str): return False - entry = saved_groups.get(group_id) + entry = saved_groups.get(group_id) if saved_groups is not None else None if not isinstance(entry, dict): return False @@ -85,7 +84,9 @@ def _eval_saved_group( condition = entry.get("condition") if not isinstance(condition, dict): return False - return evalCondition(attributes, condition, saved_groups, visited | {group_id}) + # Allocate the branch guard only when entering a condition group. + next_visited = visited | {group_id} if visited else {group_id} + return evalCondition(attributes, condition, saved_groups, next_visited) except (TypeError, ValueError, AttributeError, IndexError, RecursionError): # Malformed payloads and Python's own stack limit must not escape evaluation. return False @@ -291,7 +292,9 @@ def evalOperatorCondition( elif operator in ("$inGroup", "$notInGroup"): if not isinstance(conditionValue, str): return False - values = _saved_group_values(savedGroups, conditionValue) + values = savedGroups.get(conditionValue) if savedGroups is not None else None + if not isinstance(values, list): + values = _saved_group_values(savedGroups, conditionValue) if values is None: return False matches = isIn(values, attributeValue) diff --git a/tests/scripts/README.md b/tests/scripts/README.md index db25089..c244c73 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -9,6 +9,20 @@ Also in this directory (documented in their own docstrings): `GrowthBookClient` with sync/async sticky bucket services. Reports throughput, latency percentiles, and event-loop lag. Run with `PYTHONPATH=. python3 tests/scripts/benchmark_async_client.py`. +- `benchmark_saved_groups.py` — sync and async evaluation benchmarks for + ordinary conditions, legacy groups, and v2 list/condition groups. Checks + results before timing and emits JSON with all samples. Compare the same + workload on two checkouts with the same Python environment: + + ```bash + python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/base --legacy-only + python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/change --legacy-only + python tests/scripts/benchmark_saved_groups.py # includes v2 workloads + ``` + + Run sequentially, alternate checkout order, and repeat before drawing + conclusions. `--iterations` and `--rounds` control sample size. These are + local CPU measurements, not network or production throughput guarantees. - `check_corpus_freshness.py` — cases.json drift check vs the JS SDK (runs as its own CI job). diff --git a/tests/scripts/benchmark_saved_groups.py b/tests/scripts/benchmark_saved_groups.py new file mode 100644 index 0000000..d58d03c --- /dev/null +++ b/tests/scripts/benchmark_saved_groups.py @@ -0,0 +1,132 @@ +"""Measure condition evaluation through both public clients, without network I/O. + +Compare the same workloads on two checkouts using the same interpreter: + + python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/base --legacy-only + python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/change --legacy-only + python tests/scripts/benchmark_saved_groups.py # also measure v2 payloads + +Run comparisons sequentially and alternate checkout order to reduce interference +and thermal bias. JSON output includes the imported SDK path and every sample. +Payload loading, correctness checks, and cleanup are outside the timed region. +This measures CPU cost per evaluation; benchmark_async_client.py measures +concurrent requests with simulated sticky-bucket I/O and event-loop lag. +""" + +import argparse +import asyncio +import json +import platform +import statistics +import sys +import time +from copy import deepcopy +from pathlib import Path + + +def scenarios(): + """Return equivalent legacy/v2 conditions plus ordinary evaluation controls.""" + attributes = {"id": "u999", "plan": "pro", "country": "US", "tags": ["u998", "u999"]} + plain = {"plan": "pro", "country": "US"} + nested = {"$and": [plain, {"$not": {"plan": "free"}}, {"$or": [{"country": "CA"}, plain]}]} + cases = [ + ("default", None, {}, False), + ("experiment", None, {}, False), + ("plain-condition", plain, {}, False), + ("nested-condition", nested, {}, False), + ] + for size in (10, 1000): + values = [f"u{i}" for i in range(1000 - size, 1000)] + cases.extend([ + (f"legacy-list-{size}", {"id": {"$inGroup": "g"}}, {"g": values}, False), + (f"v2-list-{size}", {"$savedGroup": {"id": "g"}}, { + "g": {"type": "list", "attributeKey": "id", "values": values}, + }, True), + ]) + values = [f"u{i}" for i in range(1000)] + cases.extend([ + ("legacy-array-1000", {"tags": {"$inGroup": "g"}}, {"g": values}, False), + ("legacy-not-in-group", {"id": {"$notInGroup": "g"}}, {"g": ["other"]}, False), + ("v2-condition", {"$savedGroup": {"id": "g"}}, { + "g": {"type": "condition", "condition": plain}, + }, True), + ("v2-nested-condition", {"$savedGroup": {"id": "g"}}, { + "g": {"type": "condition", "condition": nested}, + }, True), + ]) + for name, condition, groups, v2 in cases: + feature = {"defaultValue": True} + if name == "experiment": + feature = {"defaultValue": False, "rules": [{"key": "bench", "variations": [True, True], "coverage": 1}]} + elif condition is not None: + feature = {"defaultValue": False, "rules": [{"condition": condition, "force": True}]} + yield name, attributes, {"features": {"flag": feature}, "savedGroups": groups}, v2 + + +async def benchmark(args): + """Load the selected checkout before importing either client.""" + sdk_path = Path(args.sdk_path).resolve() + sys.path.insert(0, str(sdk_path)) + import growthbook + from growthbook import GrowthBook, GrowthBookClient, Options, UserContext + + assert Path(growthbook.__file__).resolve().parent == sdk_path / "growthbook" + for name, attributes, payload, v2 in scenarios(): + if args.legacy_only and v2: + continue + sync_client = GrowthBook(attributes=deepcopy(attributes)) + async_client = GrowthBookClient(Options()) + user = UserContext(attributes=deepcopy(attributes)) + try: + sync_client.set_payload(payload) + await async_client.set_payload(payload) + for client_name in ("sync", "async"): + expected_source = "defaultValue" if name == "default" else "experiment" if name == "experiment" else "force" + if client_name == "sync": + result = sync_client.eval_feature("flag") + else: + result = await async_client.eval_feature("flag", user) + assert result.value is True and result.source == expected_source, (name, client_name, result.to_dict()) + + samples = [] + for round_index in range(args.rounds + 1): + count = min(args.iterations, 1000) if round_index == 0 else args.iterations + start = time.perf_counter_ns() + if client_name == "sync": + for _ in range(count): + sync_client.eval_feature("flag") + else: + for _ in range(count): + await async_client.eval_feature("flag", user) + elapsed = time.perf_counter_ns() - start + if round_index: + samples.append(elapsed / count / 1000) + print(json.dumps({ + "scenario": name, + "client": client_name, + "sdk_path": str(sdk_path), + "python": platform.python_version(), + "iterations": args.iterations, + "median_us": statistics.median(samples), + "min_us": min(samples), + "samples_us": samples, + }), flush=True) + finally: + sync_client.destroy() + await async_client.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sdk-path", default=str(Path(__file__).resolve().parents[2])) + parser.add_argument("--legacy-only", action="store_true") + parser.add_argument("--iterations", type=int, default=20000) + parser.add_argument("--rounds", type=int, default=7) + args = parser.parse_args() + if args.iterations < 1 or args.rounds < 1: + parser.error("--iterations and --rounds must be positive") + asyncio.run(benchmark(args)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_saved_groups.py b/tests/test_saved_groups.py index 728b3b3..191d5e4 100644 --- a/tests/test_saved_groups.py +++ b/tests/test_saved_groups.py @@ -1,14 +1,21 @@ """Saved group references v2 conformance and Python-specific regressions.""" +import asyncio import json +import os import sys +from base64 import b64decode, b64encode from copy import deepcopy from pathlib import Path import pytest +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from unittest.mock import AsyncMock -from growthbook import Experiment, GrowthBook, GrowthBookClient, Options, UserContext +from growthbook import Experiment, FeatureRepository, GrowthBook, GrowthBookClient, Options, UserContext, feature_repo from growthbook.core import evalCondition +from growthbook.growthbook_client import EnhancedFeatureRepository CASES = json.loads(Path(__file__).with_name("cases.json").read_text())[ @@ -243,3 +250,97 @@ async def test_switching_payload_formats_preserves_results(attributes, expected) finally: sync_client.destroy() await async_client.close() + + +def _encrypt_section(value, key): + """Encode a test payload using the SDK's AES-CBC wire format.""" + iv = os.urandom(16) + padder = padding.PKCS7(128).padder() + padded = padder.update(json.dumps(value).encode()) + padder.finalize() + encryptor = Cipher(algorithms.AES(b64decode(key)), modes.CBC(iv)).encryptor() + ciphertext = encryptor.update(padded) + encryptor.finalize() + return b64encode(iv).decode() + "." + b64encode(ciphertext).decode() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("encrypted", [False, True]) +@pytest.mark.parametrize("transport", ["set_payload", "fetch"]) +async def test_both_clients_load_and_refresh_typed_groups(mocker, encrypted, transport): + mocker.patch.object(EnhancedFeatureRepository, "_instances", {}) + key = b64encode(b"0123456789abcdef").decode() + payload = { + "features": { + "flag": {"defaultValue": False, "rules": [{ + "condition": {"$savedGroup": {"id": "eligible"}}, "force": True, + }]}, + "unsafe": {"defaultValue": False, "rules": [{ + "condition": {"id": {"$notInGroup": "bad"}}, "force": True, + }]}, + }, + "savedGroups": { + "members": {"type": "list", "attributeKey": "id", "values": ["u1"]}, + "eligible": {"type": "condition", "condition": { + "$and": [{"$savedGroup": {"id": "members"}}, {"plan": "pro"}], + }}, + "bad": None, + }, + } + wire = {} + sync_fetch = mocker.patch.object( + FeatureRepository, "_fetch_and_decode", side_effect=lambda *args: deepcopy(wire), + ) + async_fetch = mocker.patch.object( + FeatureRepository, "_fetch_and_decode_async", new_callable=AsyncMock, + side_effect=lambda *args: deepcopy(wire), + ) + mocker.patch.object(EnhancedFeatureRepository, "start_feature_refresh", new_callable=AsyncMock) + sync_client = GrowthBook(client_key="sdk-saved-sync" if transport == "fetch" else "", decryption_key=key) + async_client = GrowthBookClient(Options(client_key="sdk-saved-async" if transport == "fetch" else "", decryption_key=key)) + users = [{"id": "u1", "plan": "pro"}, {"id": "u2", "plan": "pro"}, {"id": "u1", "plan": "free"}] + experiment = Experiment(key="saved-group", variations=[0, 1], condition={"$savedGroup": {"id": "eligible"}}) + + async def check_user(attributes, member): + user = UserContext(attributes=attributes) + expected = attributes["id"] == member and attributes["plan"] == "pro" + assert await async_client.is_on("flag", user) is expected + assert await async_client.is_on("unsafe", user) is False + assert (await async_client.run(experiment, user)).inExperiment is expected + + feature_repo.clear_cache() + try: + for index, member in enumerate(("u1", "u2")): + payload["savedGroups"]["members"]["values"] = [member] + wire = ({ + "encryptedFeatures": _encrypt_section(payload["features"], key), + "encryptedSavedGroups": _encrypt_section(payload["savedGroups"], key), + } if encrypted else deepcopy(payload)) + if transport == "set_payload": + sync_client.set_payload(deepcopy(wire)) + await async_client.set_payload(deepcopy(wire)) + else: + sync_client.load_features(force_refresh=True) + if index == 0: + assert await async_client.initialize() + else: + repo = async_client._features_repository + updated = await repo.load_features_async( + "https://cdn.growthbook.io", "sdk-saved-async", key, force_refresh=True, + ) + await repo._handle_feature_update(updated) + for attributes in users: + expected = attributes["id"] == member and attributes["plan"] == "pro" + sync_client.set_attributes(dict(attributes)) + assert sync_client.is_on("flag") is expected + assert sync_client.is_on("unsafe") is False + assert sync_client.run(experiment).inExperiment is expected + await asyncio.gather(*(check_user(dict(attributes), member) for _ in range(10) for attributes in users)) + if transport == "fetch": + assert sync_fetch.call_count == 2 + assert async_fetch.await_count == 2 + else: + sync_fetch.assert_not_called() + async_fetch.assert_not_called() + finally: + sync_client.destroy() + await async_client.close() + feature_repo.clear_cache() From 7cfd09c90a4c690d167a89d9f5b813beb988ad39 Mon Sep 17 00:00:00 2001 From: Andy White Date: Fri, 25 Sep 2026 16:43:42 -0600 Subject: [PATCH 3/6] docs: record saved group performance baseline --- .../saved-group-references-v2.jsonl | 157 ++++++++++++++++++ docs/benchmarks/saved-group-references-v2.md | 119 +++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 docs/benchmarks/saved-group-references-v2.jsonl create mode 100644 docs/benchmarks/saved-group-references-v2.md diff --git a/docs/benchmarks/saved-group-references-v2.jsonl b/docs/benchmarks/saved-group-references-v2.jsonl new file mode 100644 index 0000000..6579f7a --- /dev/null +++ b/docs/benchmarks/saved-group-references-v2.jsonl @@ -0,0 +1,157 @@ +{"baseline": "e6e7773cfbe7aa00202708d3fa49bca849797602", "candidate": "75c1c1a1789dd53f55cdecf6dac23084ba8058f8", "core_sha256": "ee3481f6b31a83ef136cf7afe27e74fda010a46819b0204fe8641e9d00aec2fd", "date": "2026-09-25", "js_main": "e613b6149809c71f5155f6af6cd92a437b447c01", "kind": "metadata", "order": ["base", "final", "final", "base", "base", "final"], "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", "python": "3.13.15"} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.91525625, "min_us": 0.8912521, "python": "3.13.15", "samples_us": [0.8997062499999999, 0.93126665, 0.96367085, 0.8912521, 0.91284165, 0.91539585, 0.91525625], "scenario": "default", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 0.999075, "min_us": 0.97498125, "python": "3.13.15", "samples_us": [1.02040415, 1.0007854, 0.999075, 0.9828937499999999, 0.98577705, 0.97498125, 1.0121375000000001], "scenario": "default", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.706862500000001, "min_us": 4.6425458, "python": "3.13.15", "samples_us": [4.706862500000001, 4.8592896, 4.84730625, 4.7390979, 4.65432915, 4.6425458, 4.65135625], "scenario": "experiment", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.53742085, "min_us": 4.44055, "python": "3.13.15", "samples_us": [4.568325, 4.6216479, 4.53742085, 4.4692583500000005, 4.54255, 4.44055, 4.50575], "scenario": "experiment", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.52416875, "min_us": 1.48246875, "python": "3.13.15", "samples_us": [1.5231312499999998, 1.52603125, 1.5018270500000002, 1.48246875, 1.52416875, 1.54145625, 1.58462085], "scenario": "plain-condition", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.63176665, "min_us": 1.6008479, "python": "3.13.15", "samples_us": [1.61989375, 1.64097915, 1.6819791499999999, 1.63176665, 1.6156541500000001, 1.6008479, 1.64098125], "scenario": "plain-condition", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.7111229, "min_us": 2.6280187500000003, "python": "3.13.15", "samples_us": [2.74357915, 3.1726687499999997, 2.69193335, 2.6280187500000003, 2.6663875, 2.7111229, 2.7519583499999998], "scenario": "nested-condition", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.8125875000000002, "min_us": 2.7775396, "python": "3.13.15", "samples_us": [2.9263646, 2.7775396, 2.8125875000000002, 2.783575, 2.82102705, 2.83378955, 2.7801895500000002], "scenario": "nested-condition", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.70637705, "min_us": 1.63776875, "python": "3.13.15", "samples_us": [1.65034795, 1.63776875, 1.82824585, 1.7245583500000001, 1.76135835, 1.638075, 1.70637705], "scenario": "legacy-list-10", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7654521, "min_us": 1.75809795, "python": "3.13.15", "samples_us": [1.8243146, 1.75809795, 1.7654521, 1.77506875, 1.7876229000000001, 1.7623145999999998, 1.76419375], "scenario": "legacy-list-10", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.253514600000001, "min_us": 6.073370799999999, "python": "3.13.15", "samples_us": [6.30440625, 6.073370799999999, 6.333629200000001, 6.3787042000000005, 6.253514600000001, 6.129608350000001, 6.146525], "scenario": "legacy-list-1000", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.3326, "min_us": 6.2001958, "python": "3.13.15", "samples_us": [6.4191667, 6.357431249999999, 6.3326, 6.3213625, 6.2001958, 6.268572949999999, 6.410858350000001], "scenario": "legacy-list-1000", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.76250625, "min_us": 8.501020800000001, "python": "3.13.15", "samples_us": [8.76250625, 8.773064549999999, 8.9165521, 8.8681479, 8.57505625, 8.57900835, 8.501020800000001], "scenario": "legacy-array-1000", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.68685, "min_us": 8.4751042, "python": "3.13.15", "samples_us": [8.7025875, 8.4751042, 8.72375205, 8.68685, 8.48396045, 8.6664917, 9.10630415], "scenario": "legacy-array-1000", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6768604, "min_us": 1.6620479499999998, "python": "3.13.15", "samples_us": [1.6684583, 1.67775, 1.7091729, 1.6620479499999998, 1.68225625, 1.6768604, 1.67084375], "scenario": "legacy-not-in-group", "trial": 1} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.8142521, "min_us": 1.71752915, "python": "3.13.15", "samples_us": [1.83485415, 1.8142521, 1.8146875, 1.8506500000000001, 1.8081958500000002, 1.76027295, 1.71752915], "scenario": "legacy-not-in-group", "trial": 1} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.91474585, "min_us": 0.88725835, "python": "3.13.15", "samples_us": [0.9193875, 0.93439585, 0.91474585, 0.9085583500000001, 0.97683545, 0.9123354, 0.88725835], "scenario": "default", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 0.99360205, "min_us": 0.9698125, "python": "3.13.15", "samples_us": [0.9785458499999999, 0.9728312499999999, 1.0278146, 1.00254795, 0.99360205, 1.0011146, 0.9698125], "scenario": "default", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.6533521, "min_us": 4.38875835, "python": "3.13.15", "samples_us": [4.443745799999999, 4.38875835, 4.6533521, 4.6271375, 4.7675417, 4.67882295, 4.751027049999999], "scenario": "experiment", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.51281875, "min_us": 4.4165354, "python": "3.13.15", "samples_us": [4.74438125, 4.5975917, 4.56803125, 4.51281875, 4.44649795, 4.42695, 4.4165354], "scenario": "experiment", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.5481395500000001, "min_us": 1.4880125, "python": "3.13.15", "samples_us": [1.61680205, 1.5054146000000002, 1.57598335, 1.4880125, 1.5561125, 1.5481395500000001, 1.5113583000000002], "scenario": "plain-condition", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.5771729499999998, "min_us": 1.5627333, "python": "3.13.15", "samples_us": [1.57369165, 1.5771729499999998, 1.58562915, 1.5627333, 1.56916455, 1.6123832999999999, 1.5776396], "scenario": "plain-condition", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.69394165, "min_us": 2.60333125, "python": "3.13.15", "samples_us": [2.8280666500000002, 2.6632792, 2.69394165, 2.66217915, 2.7809479, 2.7127146, 2.60333125], "scenario": "nested-condition", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.7569979, "min_us": 2.6780999999999997, "python": "3.13.15", "samples_us": [3.2351604000000003, 3.75744165, 2.7474021, 2.7569979, 2.7303854500000004, 2.6780999999999997, 2.7830792], "scenario": "nested-condition", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6455687499999998, "min_us": 1.6046542, "python": "3.13.15", "samples_us": [1.6121646, 1.6222895499999999, 1.67638335, 1.68984795, 1.6767896, 1.6046542, 1.6455687499999998], "scenario": "legacy-list-10", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7290854, "min_us": 1.7090375, "python": "3.13.15", "samples_us": [1.7236270999999999, 1.7090375, 1.8340604, 1.74840625, 1.81980625, 1.7290854, 1.71015], "scenario": "legacy-list-10", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.2734542, "min_us": 6.0787792, "python": "3.13.15", "samples_us": [7.5588, 6.10987295, 6.0787792, 6.299487500000001, 6.200808299999999, 6.31490835, 6.2734542], "scenario": "legacy-list-1000", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.122327100000001, "min_us": 6.03718335, "python": "3.13.15", "samples_us": [6.1190708, 6.122327100000001, 6.2761729, 6.03718335, 6.0997625, 6.15894165, 6.3735520999999995], "scenario": "legacy-list-1000", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.4603375, "min_us": 8.2003542, "python": "3.13.15", "samples_us": [8.25294585, 8.25831875, 8.54996665, 8.645237499999999, 8.4603375, 8.5577625, 8.2003542], "scenario": "legacy-array-1000", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.250675, "min_us": 8.1360167, "python": "3.13.15", "samples_us": [8.345635399999999, 8.18569795, 8.1360167, 8.231854149999998, 8.47146665, 8.3660583, 8.250675], "scenario": "legacy-array-1000", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6148104, "min_us": 1.58423955, "python": "3.13.15", "samples_us": [1.66056665, 1.6008958500000001, 1.58423955, 1.6127625, 1.64599585, 1.62270625, 1.6148104], "scenario": "legacy-not-in-group", "trial": 2} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.76300835, "min_us": 1.73237915, "python": "3.13.15", "samples_us": [1.73237915, 1.76300835, 1.8029604, 1.7755562500000002, 1.75437295, 1.7917292, 1.7468458], "scenario": "legacy-not-in-group", "trial": 2} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.89704375, "min_us": 0.8823917, "python": "3.13.15", "samples_us": [0.8959520999999999, 0.89704375, 0.89418335, 0.8823917, 0.90855625, 0.89803125, 0.9011667], "scenario": "default", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.0110646, "min_us": 0.9736479, "python": "3.13.15", "samples_us": [0.9736479, 0.98921875, 1.01546665, 1.0148728999999999, 1.0110646, 0.99019585, 1.0207875], "scenario": "default", "trial": 3} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.41017085, "min_us": 4.360264600000001, "python": "3.13.15", "samples_us": [4.360264600000001, 4.4131479, 4.4001417, 4.38338125, 4.41017085, 4.4365353999999995, 4.52211455], "scenario": "experiment", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.613125, "min_us": 4.3757771000000005, "python": "3.13.15", "samples_us": [4.461525, 4.3757771000000005, 4.6635145499999995, 4.57378125, 4.6531604, 4.62615625, 4.613125], "scenario": "experiment", "trial": 3} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.5349104, "min_us": 1.4938604, "python": "3.13.15", "samples_us": [1.5166021, 1.5349104, 1.5509104, 1.5962417, 1.55605835, 1.4938604, 1.5266229], "scenario": "plain-condition", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.5997146, "min_us": 1.5532521, "python": "3.13.15", "samples_us": [1.6251167, 1.62797705, 1.5914437499999998, 1.5997146, 1.63396455, 1.595675, 1.5532521], "scenario": "plain-condition", "trial": 3} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.6744875, "min_us": 2.5694, "python": "3.13.15", "samples_us": [2.6127458499999996, 2.6744875, 2.70914585, 2.63179375, 2.67840625, 2.5694, 2.7041853999999996], "scenario": "nested-condition", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.73277705, "min_us": 2.66449375, "python": "3.13.15", "samples_us": [2.74719375, 2.7775542, 2.69385, 2.66449375, 2.73277705, 2.6922833500000003, 2.82133125], "scenario": "nested-condition", "trial": 3} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.61557915, "min_us": 1.5849208000000001, "python": "3.13.15", "samples_us": [1.61557915, 1.5978021, 1.6208125, 1.60007915, 1.6609583, 1.5849208000000001, 1.67297085], "scenario": "legacy-list-10", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7643583, "min_us": 1.6932271, "python": "3.13.15", "samples_us": [1.70834795, 1.6932271, 1.7643583, 1.801025, 1.7724916499999999, 1.7626125, 1.8133770999999999], "scenario": "legacy-list-10", "trial": 3} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.3564853999999995, "min_us": 6.20002085, "python": "3.13.15", "samples_us": [6.47531045, 6.4059875, 6.3106104, 6.3564853999999995, 6.4675083, 6.2191292, 6.20002085], "scenario": "legacy-list-1000", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.22628335, "min_us": 6.12835, "python": "3.13.15", "samples_us": [6.1662271, 6.2325625, 6.248925, 6.12835, 6.20111665, 6.22628335, 6.272925], "scenario": "legacy-list-1000", "trial": 3} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.7573875, "min_us": 8.64815415, "python": "3.13.15", "samples_us": [8.7573875, 8.64815415, 8.7726521, 8.726835450000001, 9.1262229, 8.77975, 8.73453125], "scenario": "legacy-array-1000", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.811931249999999, "min_us": 8.7077708, "python": "3.13.15", "samples_us": [8.7684625, 9.171306249999999, 8.87372085, 8.7077708, 8.77232915, 9.0740625, 8.811931249999999], "scenario": "legacy-array-1000", "trial": 3} +{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.55896875, "min_us": 1.5382271, "python": "3.13.15", "samples_us": [1.5382271, 1.5446875, 1.5879583, 1.5518771, 2.12846875, 1.8725771, 1.55896875], "scenario": "legacy-not-in-group", "trial": 3} +{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7298145999999999, "min_us": 1.71821665, "python": "3.13.15", "samples_us": [1.86085, 1.89499795, 1.7265708, 1.7415604, 1.7298145999999999, 1.71821665, 1.7265229500000001], "scenario": "legacy-not-in-group", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.9501854, "min_us": 0.9168875000000001, "python": "3.13.15", "samples_us": [0.9369895500000001, 0.95731665, 0.9501854, 0.9403937499999999, 1.04558125, 1.0520895499999998, 0.9168875000000001], "scenario": "default", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.0163083, "min_us": 1.00188335, "python": "3.13.15", "samples_us": [1.0407875, 1.0366292000000001, 1.0113792, 1.0163083, 1.1125604, 1.00188335, 1.0048771], "scenario": "default", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.5898417, "min_us": 4.4328083, "python": "3.13.15", "samples_us": [4.4328083, 4.52304375, 5.08686665, 4.585875, 4.5898417, 4.7070667, 4.8008021], "scenario": "experiment", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.6828708500000005, "min_us": 4.57466665, "python": "3.13.15", "samples_us": [4.74911875, 4.6135292, 4.57466665, 4.6828708500000005, 4.71356455, 4.7073395499999995, 4.60502915], "scenario": "experiment", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.5812728999999999, "min_us": 1.5340625, "python": "3.13.15", "samples_us": [1.5812728999999999, 1.5340625, 1.6008521, 1.61313125, 1.5955896, 1.55369585, 1.5746875], "scenario": "plain-condition", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.61463335, "min_us": 1.60075415, "python": "3.13.15", "samples_us": [1.65883335, 1.61463335, 1.60075415, 1.61993125, 1.6063646, 1.6028396, 1.62542085], "scenario": "plain-condition", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.78591875, "min_us": 2.72218125, "python": "3.13.15", "samples_us": [2.86320415, 2.78591875, 2.76523125, 2.7335021, 2.72218125, 2.78785205, 2.79923335], "scenario": "nested-condition", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.9130229, "min_us": 2.8123395500000004, "python": "3.13.15", "samples_us": [2.94036045, 2.9741999999999997, 2.9130229, 2.83405205, 2.8909499999999997, 2.9769396, 2.8123395500000004], "scenario": "nested-condition", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.734125, "min_us": 1.691375, "python": "3.13.15", "samples_us": [1.6941708000000002, 1.691375, 1.734125, 1.7142125000000001, 1.78829165, 1.7566041499999998, 1.82087085], "scenario": "legacy-list-10", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.89686875, "min_us": 1.8760541499999999, "python": "3.13.15", "samples_us": [1.89070625, 1.96609795, 1.9297396, 1.905875, 1.8906229, 1.8760541499999999, 1.89686875], "scenario": "legacy-list-10", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6565375, "min_us": 1.64021875, "python": "3.13.15", "samples_us": [1.6439896, 1.64021875, 1.665075, 1.6427583, 1.6859604, 1.6565375, 1.6599146], "scenario": "v2-list-10", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.68880625, "min_us": 1.6711395500000001, "python": "3.13.15", "samples_us": [1.6783458, 1.68416045, 2.1187332999999997, 1.73556045, 1.77300625, 1.68880625, 1.6711395500000001], "scenario": "v2-list-10", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.19900625, "min_us": 6.16515, "python": "3.13.15", "samples_us": [6.16515, 6.19550835, 6.1751854, 6.19900625, 6.20026875, 6.3707875000000005, 6.2020979], "scenario": "legacy-list-1000", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.29534375, "min_us": 6.2655125, "python": "3.13.15", "samples_us": [6.29534375, 6.39862085, 6.4570958, 6.3455979, 6.2655125, 6.266958300000001, 6.2697978999999995], "scenario": "legacy-list-1000", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.04683125, "min_us": 5.91361665, "python": "3.13.15", "samples_us": [6.045649999999999, 6.03578335, 6.16063335, 6.04683125, 6.1772124999999996, 6.135493749999999, 5.91361665], "scenario": "v2-list-1000", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.24695415, "min_us": 6.125920850000001, "python": "3.13.15", "samples_us": [6.14290835, 6.28399375, 6.228868749999999, 6.3402083000000005, 7.2605333, 6.125920850000001, 6.24695415], "scenario": "v2-list-1000", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 9.22092915, "min_us": 8.8211271, "python": "3.13.15", "samples_us": [8.963700000000001, 8.8211271, 9.22092915, 9.45024165, 9.253768749999999, 9.2312396, 8.9192104], "scenario": "legacy-array-1000", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.89325835, "min_us": 8.77605835, "python": "3.13.15", "samples_us": [8.80182915, 9.016825, 8.89325835, 8.77605835, 8.852118749999999, 8.964775, 8.99563545], "scenario": "legacy-array-1000", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.73110835, "min_us": 1.70992295, "python": "3.13.15", "samples_us": [1.73110835, 1.84421875, 1.9713270999999999, 1.7134729, 1.8164854, 1.7197770499999998, 1.70992295], "scenario": "legacy-not-in-group", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.8336292, "min_us": 1.7653333, "python": "3.13.15", "samples_us": [1.81821875, 1.8336292, 1.8361958, 1.85109375, 1.8723937499999999, 1.7653333, 1.8048333], "scenario": "legacy-not-in-group", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.82614375, "min_us": 1.75386875, "python": "3.13.15", "samples_us": [1.86775, 1.8218417, 1.8886979, 1.84129795, 1.82614375, 1.7737271000000001, 1.75386875], "scenario": "v2-condition", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.89778125, "min_us": 1.8499375, "python": "3.13.15", "samples_us": [1.8607979499999998, 2.0668625, 1.89778125, 1.86859375, 1.9527750000000001, 1.8499375, 1.9346312499999998], "scenario": "v2-condition", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 3.0239895999999997, "min_us": 2.95849375, "python": "3.13.15", "samples_us": [3.0437354, 2.95849375, 3.1286583500000003, 3.0239895999999997, 3.0745603999999997, 2.9833333499999997, 2.96141875], "scenario": "v2-nested-condition", "trial": 1} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 3.0929708000000002, "min_us": 3.0620666500000002, "python": "3.13.15", "samples_us": [3.1599625000000002, 3.1578542, 3.1047125, 3.0620666500000002, 3.07316455, 3.0859978999999997, 3.0929708000000002], "scenario": "v2-nested-condition", "trial": 1} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.95605, "min_us": 0.9466687500000001, "python": "3.13.15", "samples_us": [0.95022085, 0.9466687500000001, 0.97379165, 0.98290625, 0.9506042, 0.95605, 0.9765979], "scenario": "default", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.0587208499999998, "min_us": 1.0116083, "python": "3.13.15", "samples_us": [1.0116083, 1.03121875, 1.0647375, 1.06830625, 1.0706396, 1.0587208499999998, 1.03341875], "scenario": "default", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.6418187500000005, "min_us": 4.5037291999999995, "python": "3.13.15", "samples_us": [4.78569375, 4.7059896, 4.6778249999999995, 4.6418187500000005, 4.56280415, 4.57425, 4.5037291999999995], "scenario": "experiment", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 5.51788125, "min_us": 4.52385625, "python": "3.13.15", "samples_us": [4.52385625, 4.603345849999999, 4.632860399999999, 5.93967295, 6.853202100000001, 14.9319854, 5.51788125], "scenario": "experiment", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6151396, "min_us": 1.58689165, "python": "3.13.15", "samples_us": [1.58689165, 1.6851729, 1.6809979499999999, 1.6151396, 1.6202479, 1.59508125, 1.60548125], "scenario": "plain-condition", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7059396, "min_us": 1.6810916500000002, "python": "3.13.15", "samples_us": [1.7028125, 1.6810916500000002, 1.71805835, 1.6813271, 1.7059396, 1.72374165, 1.91619375], "scenario": "plain-condition", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.78348125, "min_us": 2.71338545, "python": "3.13.15", "samples_us": [3.03188125, 2.79590205, 2.72081875, 2.71338545, 2.78348125, 2.75510415, 2.81181875], "scenario": "nested-condition", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.8565125, "min_us": 2.78348125, "python": "3.13.15", "samples_us": [2.8702104, 2.8565125, 2.8226457999999996, 2.79165, 3.1992875, 2.78348125, 2.9433312500000004], "scenario": "nested-condition", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.67728335, "min_us": 1.6441375, "python": "3.13.15", "samples_us": [1.6441375, 1.7104229499999999, 1.71604585, 1.67728335, 1.6633978999999999, 1.70024375, 1.6464375], "scenario": "legacy-list-10", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.80724165, "min_us": 1.7681667, "python": "3.13.15", "samples_us": [1.77194795, 1.7681667, 1.8172292, 1.80724165, 1.8394458, 1.80647295, 1.8273167000000001], "scenario": "legacy-list-10", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.55355, "min_us": 1.5313208, "python": "3.13.15", "samples_us": [1.55355, 1.5453875000000001, 1.6414021, 1.54465625, 1.5939396, 1.56821665, 1.5313208], "scenario": "v2-list-10", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.6804229, "min_us": 1.63771875, "python": "3.13.15", "samples_us": [1.6982562499999998, 1.6722271, 1.72202085, 1.63771875, 1.6804229, 1.73258125, 1.6642021], "scenario": "v2-list-10", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.17611875, "min_us": 6.0599729, "python": "3.13.15", "samples_us": [6.0599729, 6.11313335, 6.35407915, 6.17611875, 6.2387833, 6.1717125, 6.1874771], "scenario": "legacy-list-1000", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.23740835, "min_us": 6.1702104, "python": "3.13.15", "samples_us": [6.23740835, 6.17885415, 6.29840205, 6.1702104, 6.190608350000001, 6.260325, 6.32633125], "scenario": "legacy-list-1000", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.0269271, "min_us": 5.941304199999999, "python": "3.13.15", "samples_us": [6.0269271, 6.0803896, 6.0508500000000005, 5.941304199999999, 5.9605042, 5.94300415, 6.0920917], "scenario": "v2-list-1000", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.25116665, "min_us": 6.0390229, "python": "3.13.15", "samples_us": [6.25116665, 6.16399795, 6.3913604500000005, 6.37139375, 6.3981396, 6.0390229, 6.176960449999999], "scenario": "v2-list-1000", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.4854146, "min_us": 8.3594167, "python": "3.13.15", "samples_us": [8.51044585, 8.60761875, 8.506662500000001, 8.39770835, 8.36065415, 8.3594167, 8.4854146], "scenario": "legacy-array-1000", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.772197949999999, "min_us": 8.60236665, "python": "3.13.15", "samples_us": [8.772197949999999, 8.85310415, 8.7743229, 8.63303335, 8.693689549999998, 8.60236665, 9.15104165], "scenario": "legacy-array-1000", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6505958, "min_us": 1.57345625, "python": "3.13.15", "samples_us": [1.59058955, 1.63530835, 1.68686045, 1.6505958, 1.6950875, 1.6640875000000002, 1.57345625], "scenario": "legacy-not-in-group", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.74729165, "min_us": 1.66358125, "python": "3.13.15", "samples_us": [1.74729165, 1.7421854, 1.7461146, 1.66358125, 1.8213521000000001, 1.7739604, 1.7948521000000002], "scenario": "legacy-not-in-group", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.7503604, "min_us": 1.7275125, "python": "3.13.15", "samples_us": [1.72860835, 1.7714520500000002, 1.7415916999999999, 1.7275125, 1.76431875, 1.76111875, 1.7503604], "scenario": "v2-condition", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.8784792, "min_us": 1.8351646000000001, "python": "3.13.15", "samples_us": [1.8351646000000001, 1.8472457999999998, 1.8446937499999998, 1.91453125, 1.9891937499999999, 1.9292083, 1.8784792], "scenario": "v2-condition", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.9533604, "min_us": 2.9218312500000003, "python": "3.13.15", "samples_us": [2.9513896, 2.9304417, 2.9751792, 2.9218312500000003, 2.9664708500000003, 3.03365, 2.9533604], "scenario": "v2-nested-condition", "trial": 2} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 3.0615124999999996, "min_us": 3.020025, "python": "3.13.15", "samples_us": [3.2403625000000003, 3.0282583, 3.0478521, 3.0615124999999996, 3.0631875, 3.14950625, 3.020025], "scenario": "v2-nested-condition", "trial": 2} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.8945125, "min_us": 0.88124375, "python": "3.13.15", "samples_us": [0.93894585, 0.8945125, 0.8949771, 0.88124375, 0.9173666500000001, 0.8933917, 0.88315], "scenario": "default", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 0.9990229500000001, "min_us": 0.9580187499999999, "python": "3.13.15", "samples_us": [0.96249585, 0.9580187499999999, 0.9990229500000001, 1.01775415, 1.0010375, 1.00807915, 0.9884562499999999], "scenario": "default", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.43501875, "min_us": 4.4065458500000005, "python": "3.13.15", "samples_us": [4.41203125, 4.4065458500000005, 4.51653335, 4.42620625, 4.46205625, 4.45600205, 4.43501875], "scenario": "experiment", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.41778335, "min_us": 4.3800021, "python": "3.13.15", "samples_us": [4.41778335, 4.4563729499999996, 4.4309229, 4.3800021, 4.4289, 4.41570835, 4.4085125], "scenario": "experiment", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.51005, "min_us": 1.489, "python": "3.13.15", "samples_us": [1.489, 1.5491041499999998, 1.5155146, 1.51005, 1.4986062500000001, 1.4917208499999999, 1.5106791499999999], "scenario": "plain-condition", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.6117249999999999, "min_us": 1.5906229, "python": "3.13.15", "samples_us": [1.5906229, 1.5932875, 1.59845625, 1.6222771, 1.6117249999999999, 1.61611455, 1.65730625], "scenario": "plain-condition", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.67414585, "min_us": 2.64751875, "python": "3.13.15", "samples_us": [2.6580812500000004, 2.71510835, 2.6937271000000003, 2.66913125, 2.747275, 2.67414585, 2.64751875], "scenario": "nested-condition", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.7658354, "min_us": 2.7532604500000004, "python": "3.13.15", "samples_us": [2.7532604500000004, 2.76299375, 2.78788125, 2.7806937499999997, 2.7658354, 2.7633833, 2.7867646], "scenario": "nested-condition", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.70064795, "min_us": 1.68446875, "python": "3.13.15", "samples_us": [1.70064795, 1.7005875000000001, 1.68446875, 1.8260416499999998, 1.7762833, 1.7086854, 1.68660625], "scenario": "legacy-list-10", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.79745625, "min_us": 1.7823646000000002, "python": "3.13.15", "samples_us": [1.97803125, 2.0591354, 1.7883896, 1.78519375, 1.79745625, 1.83838335, 1.7823646000000002], "scenario": "legacy-list-10", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.56695, "min_us": 1.5335999999999999, "python": "3.13.15", "samples_us": [1.5658687500000001, 1.6552979, 1.60516875, 1.53732085, 1.5938687500000002, 1.56695, 1.5335999999999999], "scenario": "v2-list-10", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.6390021, "min_us": 1.62625, "python": "3.13.15", "samples_us": [1.6390021, 1.6351875, 1.65627705, 1.62625, 1.7159271, 1.6360208500000002, 1.64445835], "scenario": "v2-list-10", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.34747915, "min_us": 6.14897915, "python": "3.13.15", "samples_us": [6.22051875, 6.4223646, 6.34747915, 6.29253125, 6.3692083, 6.4708771, 6.14897915], "scenario": "legacy-list-1000", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.3347375, "min_us": 6.18475415, "python": "3.13.15", "samples_us": [6.3370271, 6.3347375, 6.20884375, 6.18475415, 6.39333125, 6.374335449999999, 6.2573604], "scenario": "legacy-list-1000", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.1611396, "min_us": 5.86745835, "python": "3.13.15", "samples_us": [6.2267792, 5.86745835, 6.1510875, 6.332908349999999, 6.18880415, 6.1611396, 6.0496875], "scenario": "v2-list-1000", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.3854229, "min_us": 6.2034708499999995, "python": "3.13.15", "samples_us": [6.35958335, 7.559699999999999, 6.634412500000001, 6.614266649999999, 6.3854229, 6.2174521, 6.2034708499999995], "scenario": "v2-list-1000", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.3610521, "min_us": 8.24090835, "python": "3.13.15", "samples_us": [8.617368749999999, 8.338177100000001, 8.33557915, 8.3610521, 8.381060450000001, 8.24090835, 8.5070125], "scenario": "legacy-array-1000", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.52853545, "min_us": 8.3322021, "python": "3.13.15", "samples_us": [8.59360625, 8.4152375, 8.52304585, 8.661845849999999, 8.52853545, 8.723395850000001, 8.3322021], "scenario": "legacy-array-1000", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6642875, "min_us": 1.64457915, "python": "3.13.15", "samples_us": [1.6627979499999999, 1.7190208500000002, 1.71021875, 1.6642875, 1.6580082999999999, 1.64457915, 1.6694979], "scenario": "legacy-not-in-group", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7586104, "min_us": 1.7374625000000001, "python": "3.13.15", "samples_us": [1.823375, 1.7524771000000001, 1.7588062500000001, 1.76736045, 1.74952915, 1.7374625000000001, 1.7586104], "scenario": "legacy-not-in-group", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.72567705, "min_us": 1.70137085, "python": "3.13.15", "samples_us": [1.7205979500000002, 1.72473335, 1.7927354, 1.8617854500000002, 1.7502875, 1.70137085, 1.72567705], "scenario": "v2-condition", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.86705415, "min_us": 1.82103125, "python": "3.13.15", "samples_us": [1.87314165, 1.82103125, 1.8302104, 1.86801665, 1.8680750000000002, 1.86705415, 1.85595625], "scenario": "v2-condition", "trial": 3} +{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.99215, "min_us": 2.88095205, "python": "3.13.15", "samples_us": [2.88095205, 2.88198335, 2.99215, 3.0952542, 3.07332915, 2.9766624999999998, 3.0010583499999997], "scenario": "v2-nested-condition", "trial": 3} +{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 3.1180896, "min_us": 3.0551666500000003, "python": "3.13.15", "samples_us": [3.08345415, 3.1180896, 3.14001455, 3.09824375, 3.1351999999999998, 3.1388625, 3.0551666500000003], "scenario": "v2-nested-condition", "trial": 3} +{"checkout": "base", "kind": "existing_sync", "scenario": "default-value", "trial": 1, "us": 0.93} +{"checkout": "base", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 1, "us": 4.166} +{"checkout": "base", "kind": "existing_sync", "scenario": "default-value", "trial": 2, "us": 0.917} +{"checkout": "base", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 2, "us": 4.146} +{"checkout": "base", "kind": "existing_sync", "scenario": "default-value", "trial": 3, "us": 0.937} +{"checkout": "base", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 3, "us": 4.096} +{"checkout": "final", "kind": "existing_sync", "scenario": "default-value", "trial": 1, "us": 0.915} +{"checkout": "final", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 1, "us": 4.145} +{"checkout": "final", "kind": "existing_sync", "scenario": "default-value", "trial": 2, "us": 0.911} +{"checkout": "final", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 2, "us": 4.15} +{"checkout": "final", "kind": "existing_sync", "scenario": "default-value", "trial": 3, "us": 0.904} +{"checkout": "final", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 3, "us": 4.081} +{"checkout": "base", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 5.26, "loop_lag_mean_ms": 5.263, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 80468.9, "trial": 1} +{"checkout": "base", "eval_p50_ms": 17.48, "eval_p95_ms": 18.51, "eval_p99_ms": 19.18, "kind": "existing_async", "loop_lag_max_ms": 0.85, "loop_lag_mean_ms": 0.189, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 5560.6, "trial": 1} +{"checkout": "base", "eval_p50_ms": 3.54, "eval_p95_ms": 4.41, "eval_p99_ms": 4.68, "kind": "existing_async", "loop_lag_max_ms": 1.48, "loop_lag_mean_ms": 0.705, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 24549.1, "trial": 1} +{"checkout": "base", "eval_p50_ms": 2.42, "eval_p95_ms": 2.78, "eval_p99_ms": 2.83, "kind": "existing_async", "loop_lag_max_ms": 0.93, "loop_lag_mean_ms": 0.662, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 33028.7, "trial": 1} +{"checkout": "base", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.51, "loop_lag_mean_ms": 3.513, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 95458.9, "trial": 2} +{"checkout": "base", "eval_p50_ms": 20.53, "eval_p95_ms": 22.06, "eval_p99_ms": 22.74, "kind": "existing_async", "loop_lag_max_ms": 0.67, "loop_lag_mean_ms": 0.204, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4736.3, "trial": 2} +{"checkout": "base", "eval_p50_ms": 3.44, "eval_p95_ms": 3.93, "eval_p99_ms": 4.1, "kind": "existing_async", "loop_lag_max_ms": 1.26, "loop_lag_mean_ms": 0.77, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26321.0, "trial": 2} +{"checkout": "base", "eval_p50_ms": 2.09, "eval_p95_ms": 2.92, "eval_p99_ms": 3.03, "kind": "existing_async", "loop_lag_max_ms": 1.25, "loop_lag_mean_ms": 0.677, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 37538.6, "trial": 2} +{"checkout": "base", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.5, "loop_lag_mean_ms": 3.504, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 95568.4, "trial": 3} +{"checkout": "base", "eval_p50_ms": 19.87, "eval_p95_ms": 21.62, "eval_p99_ms": 22.34, "kind": "existing_async", "loop_lag_max_ms": 0.76, "loop_lag_mean_ms": 0.151, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4923.1, "trial": 3} +{"checkout": "base", "eval_p50_ms": 3.39, "eval_p95_ms": 3.77, "eval_p99_ms": 3.91, "kind": "existing_async", "loop_lag_max_ms": 1.15, "loop_lag_mean_ms": 0.82, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26758.0, "trial": 3} +{"checkout": "base", "eval_p50_ms": 2.06, "eval_p95_ms": 2.72, "eval_p99_ms": 3.14, "kind": "existing_async", "loop_lag_max_ms": 1.18, "loop_lag_mean_ms": 0.925, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 37858.2, "trial": 3} +{"checkout": "final", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.45, "loop_lag_mean_ms": 3.447, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 95855.1, "trial": 1} +{"checkout": "final", "eval_p50_ms": 20.02, "eval_p95_ms": 21.84, "eval_p99_ms": 22.64, "kind": "existing_async", "loop_lag_max_ms": 1.04, "loop_lag_mean_ms": 0.201, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4813.8, "trial": 1} +{"checkout": "final", "eval_p50_ms": 3.59, "eval_p95_ms": 4.15, "eval_p99_ms": 4.36, "kind": "existing_async", "loop_lag_max_ms": 1.36, "loop_lag_mean_ms": 0.618, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 25006.4, "trial": 1} +{"checkout": "final", "eval_p50_ms": 2.45, "eval_p95_ms": 2.84, "eval_p99_ms": 2.95, "kind": "existing_async", "loop_lag_max_ms": 1.09, "loop_lag_mean_ms": 0.774, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 32379.6, "trial": 1} +{"checkout": "final", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.35, "loop_lag_mean_ms": 3.354, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 96738.7, "trial": 2} +{"checkout": "final", "eval_p50_ms": 20.56, "eval_p95_ms": 22.05, "eval_p99_ms": 23.08, "kind": "existing_async", "loop_lag_max_ms": 1.14, "loop_lag_mean_ms": 0.255, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4745.5, "trial": 2} +{"checkout": "final", "eval_p50_ms": 3.4, "eval_p95_ms": 3.83, "eval_p99_ms": 3.99, "kind": "existing_async", "loop_lag_max_ms": 1.09, "loop_lag_mean_ms": 0.749, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26598.5, "trial": 2} +{"checkout": "final", "eval_p50_ms": 2.12, "eval_p95_ms": 3.11, "eval_p99_ms": 3.18, "kind": "existing_async", "loop_lag_max_ms": 1.26, "loop_lag_mean_ms": 0.765, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 36426.1, "trial": 2} +{"checkout": "final", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.02, "kind": "existing_async", "loop_lag_max_ms": 4.0, "loop_lag_mean_ms": 4.001, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 89130.2, "trial": 3} +{"checkout": "final", "eval_p50_ms": 19.65, "eval_p95_ms": 21.25, "eval_p99_ms": 22.48, "kind": "existing_async", "loop_lag_max_ms": 1.21, "loop_lag_mean_ms": 0.198, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4932.5, "trial": 3} +{"checkout": "final", "eval_p50_ms": 3.41, "eval_p95_ms": 4.12, "eval_p99_ms": 4.49, "kind": "existing_async", "loop_lag_max_ms": 1.08, "loop_lag_mean_ms": 0.581, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26272.2, "trial": 3} +{"checkout": "final", "eval_p50_ms": 2.06, "eval_p95_ms": 2.58, "eval_p99_ms": 2.67, "kind": "existing_async", "loop_lag_max_ms": 1.22, "loop_lag_mean_ms": 0.818, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 38236.0, "trial": 3} diff --git a/docs/benchmarks/saved-group-references-v2.md b/docs/benchmarks/saved-group-references-v2.md new file mode 100644 index 0000000..b58dd09 --- /dev/null +++ b/docs/benchmarks/saved-group-references-v2.md @@ -0,0 +1,119 @@ +# Saved group references v2 verification + +Measured on 2026-09-25. No material performance regression was detected in the +tested existing workloads after deferring cycle-set allocation and keeping legacy +arrays on a direct lookup path. This is a local comparison, not proof of zero +regression on every workload or interpreter. + +## Reference and correctness + +- Implementation prompt: [Notion](https://app.notion.com/p/3dda857e74a0808499d6cfba00a18196). +- Merged reference: [growthbook#6942](https://github.com/growthbook/growthbook/pull/6942). +- Checked monorepo `main` at `e613b6149809c71f5155f6af6cd92a437b447c01`. +- `mongrule.ts`, `core.ts`, and the JS corpus are identical to reference commit + `023c21040b8878650f0e2cf0eb7ca44ed4b6d2f1`, which the Python port used. +- Corpus comparison: spec 0.9.0, no missing or drifted cases; 41 Python-local extras. +- Both clients run all feature and experiment v2 cases, including prerequisites. + Additional tests cover plaintext/encrypted payload injection and fetching, + refreshed membership, malformed groups, and concurrent async evaluations. +- Final verification: 1,154 tests passed, including the MyPy and Pyright typing + regressions. Standalone MyPy, Pyright, and CI's blocking lint check also pass. + Pytest reports one existing `event_loop_policy` deprecation warning. + +The updated prompt permits capability registration before publication only with +`prerelease: true` on a new version. Remove that flag and regenerate +`CAPABILITIES.md` when publishing. Python has no such entry yet. Remote evaluation +depends on updating the proxy's JS SDK dependency; it does not require a separate +operator implementation in the proxy. + +## Method + +- Baseline: PR parent `e6e7773cfbe7aa00202708d3fa49bca849797602`. +- Candidate: `75c1c1a1789dd53f55cdecf6dac23084ba8058f8`. +- Python 3.13.15; macOS-26.6.2-arm64-arm-64bit-Mach-O. Same interpreter and installed dependencies. +- Three independent runs per checkout, ordered base/change, change/base, + base/change. All benchmark processes ran sequentially, without concurrent tests. +- Targeted benchmark: 1,000 warmup evaluations, then seven samples of 20,000 + evaluations per client/workload per run. Reported values are medians of the + three per-run medians. Client construction, payload loading, result checks, + and cleanup are excluded. Both value and result source are checked before timing. +- Existing sync benchmark: best of five samples of 100,000 evaluations per run; + report the median of the three runs. +- Existing async benchmark: 1,000 requests, concurrency 100, simulated 1 ms sticky + storage I/O; report the median of three runs. Each request evaluates two features. +- Raw samples and candidate source hash: [JSONL](saved-group-references-v2.jsonl). + +## Existing behavior: targeted benchmark + +Times are microseconds per evaluation; positive changes mean slower. + +| Workload | Sync base | Sync change | Delta | Async base | Async change | Delta | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| default | 0.915 | 0.950 | +3.9% | 0.999 | 1.016 | +1.7% | +| experiment | 4.653 | 4.590 | -1.4% | 4.537 | 4.683 | +3.2% | +| plain-condition | 1.535 | 1.581 | +3.0% | 1.600 | 1.615 | +0.9% | +| nested-condition | 2.694 | 2.783 | +3.3% | 2.757 | 2.857 | +3.6% | +| legacy-list-10 | 1.646 | 1.701 | +3.3% | 1.764 | 1.807 | +2.4% | +| legacy-list-1000 | 6.273 | 6.199 | -1.2% | 6.226 | 6.295 | +1.1% | +| legacy-array-1000 | 8.757 | 8.485 | -3.1% | 8.687 | 8.772 | +1.0% | +| legacy-not-in-group | 1.615 | 1.664 | +3.1% | 1.763 | 1.759 | -0.2% | + +All changes were between -3.1% and +3.9%. Even the unchanged default-value +control varied by +3.9%, so differences of this size should not be interpreted +as a proven speedup or slowdown. The initial implementation showed about ++0.13 µs (+7.5%) in sync `$notInGroup`; the final comparison is +0.05 µs (+3.1%). + +## V2 payload cost + +These compare equivalent decisions on the candidate checkout: a v2 list vs. +the same legacy list, or a v2 condition group vs. its inline condition. +The old SDK cannot evaluate v2 references, so timing its no-match path would +produce a misleading comparison. + +| Equivalent workload | Client | Legacy/inline µs | V2 µs | Added µs | +| --- | --- | ---: | ---: | ---: | +| v2-list-10 | sync | 1.701 | 1.567 | -0.134 | +| v2-list-10 | async | 1.807 | 1.680 | -0.127 | +| v2-list-1000 | sync | 6.199 | 6.047 | -0.152 | +| v2-list-1000 | async | 6.295 | 6.251 | -0.044 | +| v2-condition | sync | 1.581 | 1.750 | +0.169 | +| v2-condition | async | 1.615 | 1.878 | +0.264 | +| v2-nested-condition | sync | 2.783 | 2.992 | +0.209 | +| v2-nested-condition | async | 2.857 | 3.093 | +0.236 | + +## Existing benchmark scripts + +| Sync workload | Base µs | Change µs | Delta | +| --- | ---: | ---: | ---: | +| default-value | 0.930 | 0.911 | -2.0% | +| experiment-rule | 4.146 | 4.145 | -0.0% | + +| Async workload | Base req/s | Change req/s | Throughput delta | Base p95 ms | Change p95 ms | +| --- | ---: | ---: | ---: | ---: | ---: | +| no-sticky-service | 95,458.9 | 95,855.1 | +0.4% | 0.01 | 0.01 | +| sync-sticky-service | 4,923.1 | 4,813.8 | -2.2% | 21.62 | 21.84 | +| async-sticky-service | 26,321.0 | 26,272.2 | -0.2% | 3.93 | 4.12 | +| async-hot-user | 37,538.6 | 36,426.1 | -3.0% | 2.78 | 2.84 | + +Async throughput differences were within about 3%. Event-loop lag measurements +are retained in the raw data; short runs and OS scheduling make maxima noisy. +These existing scripts do not exercise saved-group conditions, which is why +the targeted benchmark is included. + +## Reproduce + +Activate the development environment. Extract or check out the baseline separately, +then run the same benchmark script against each SDK path: + +```bash +python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/baseline --legacy-only +python tests/scripts/benchmark_saved_groups.py --sdk-path "$PWD" --legacy-only +python tests/scripts/benchmark_saved_groups.py # includes v2 workloads +python tests/scripts/benchmark_eval_overhead.py 100000 +PYTHONPATH=. python tests/scripts/benchmark_async_client.py +``` + +For the existing scripts, run from each checkout with the same absolute path to +the virtual environment's Python, and set `PYTHONPATH=.` so editable-install +resolution does not accidentally benchmark the wrong checkout. Alternate the +checkout order and repeat; do not run the two benchmarks concurrently. From 0adc1181633183f0ce253fc40794237d1fec192d Mon Sep 17 00:00:00 2001 From: Andy White Date: Mon, 28 Sep 2026 15:57:31 -0600 Subject: [PATCH 4/6] docs: condense saved group documentation and benchmarks --- CONTRIBUTING.md | 334 +++--------------- README.md | 26 +- .../saved-group-references-v2.jsonl | 157 -------- docs/benchmarks/saved-group-references-v2.md | 139 ++------ tests/scripts/README.md | 17 +- 5 files changed, 94 insertions(+), 579 deletions(-) delete mode 100644 docs/benchmarks/saved-group-references-v2.jsonl diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f67851..2813451 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,227 +1,31 @@ -# Contributing Guide +# Contributing -We welcome all contributions! +## Local setup -This repo is the official GrowthBook SDK for Python — a client library for -evaluating feature flags and running experiments, with synchronous and async -clients. - -## Requirements - -- **Python 3.9+**, as declared in `pyproject.toml` and `setup.py`. CI tests - 3.9–3.13; the examples below use 3.13. Keep library code compatible with 3.9. -- **Make** for the convenience commands below. You can also invoke `pytest`, - `mypy`, and `flake8` directly from the activated environment. -- **Node.js and npm** only for the optional local Pyright checks. - -`uv` is optional. There is no dependency lockfile; CI installs from -`requirements.txt` and `requirements_dev.txt`, then installs the package. - -## Getting started - -Fork the repo, or clone directly if you have write access: - -```bash -git clone git@github.com:growthbook/growthbook-python.git -cd growthbook-python -``` - -Create a virtual environment and install the package in editable mode with -the development dependencies: +Python 3.9+ is supported; CI tests 3.9–3.13. From a cloned checkout: ```bash python3.13 -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip python -m pip install -e . -r requirements.txt -r requirements_dev.txt -make test -make type-check -``` - -If you use `uv`, the equivalent setup is: - -```bash -uv venv --python 3.13 .venv -source .venv/bin/activate -uv pip install -e . -r requirements.txt -r requirements_dev.txt -``` - -Activate `.venv` in each new terminal with `source .venv/bin/activate`. Configure -your editor to use `.venv/bin/python`. Changes to `growthbook/` are available -immediately without reinstalling the package. - -The unit tests do not need GrowthBook credentials or a running server. Get a -passing test and type-check baseline before changing code. - -## Writing code - -### Layout - -| Path | Contents | -| --- | --- | -| `growthbook/core.py` | Shared feature and experiment evaluation, targeting conditions, hashing, and bucketing | -| `growthbook/growthbook.py` | The sync `GrowthBook` client, feature repository, caching, SSE, and decryption | -| `growthbook/growthbook_client.py` | The async `GrowthBookClient`, shared feature repository, refreshes, and per-user evaluation | -| `growthbook/common_types.py` | Configuration, user and evaluation contexts, results, callback protocols, and sticky bucket interfaces | -| `growthbook/plugins/` | Tracking and request-context plugins | -| `growthbook/codegen.py` | Generator for typed clients built from feature definitions | -| `growthbook/__init__.py`, `growthbook/py.typed` | Public exports, package version, and the typed-package marker | -| `tests/test_*.py`, `tests/conftest.py` | Tests and shared fixtures | -| `tests/cases.json` | Cross-SDK conformance corpus, including Python-local cases | -| `tests/typing/`, `tests/codegen/` | Typing regression cases and code-generation fixtures | -| `tests/scripts/` | Corpus checks, benchmarks, and integration verification scripts | - -### Public API and typing - -`growthbook/__init__.py` lists the package's public exports in `__all__`. -Callers also use the documented client methods and configuration types. Discuss -changes to signatures, defaults, or return types in an issue before implementing -them; type-checking behavior is part of the API too. - -The package ships inline type annotations through `py.typed`. Add annotations -to new library functions and keep public types importable and resolvable at -runtime. When changing typing behavior, update the positive and negative cases -in `tests/typing/` and run both MyPy and Pyright — see -[Type checking](#type-checking). - -The Python 3.9 floor applies even if your local interpreter is newer. Use -`typing_extensions` where a newer typing feature needs a backport. Changes to -the minimum Python version should be discussed explicitly. - -### SDK parity - -GrowthBook SDKs should make identical decisions given identical inputs. The -[JavaScript SDK](https://github.com/growthbook/growthbook/tree/main/packages/sdk-js) -is the reference implementation. When changing evaluation, compare the -corresponding JS behavior, especially hashing, missing values, numeric types, -and targeting comparisons where Python semantics differ. - -Describe deliberate divergences in the PR. A change to which users match a -targeting rule can change live audiences when an application upgrades. - -### Sync, async, and shared state - -`GrowthBook` stores user attributes on the instance; web applications normally -create one per request and call `destroy()` afterward. `GrowthBookClient` is -reused across requests and receives a separate `UserContext` for each user. -Call `await client.close()` at shutdown, or use its async context manager. - -Both clients use `core.py`, but have separate loading and lifecycle code. -Cover both clients when changing shared behavior. In async code, avoid blocking -the event loop and test cancellation and cleanup of any background work you add. - -Feature repositories and caches can be shared. In particular, -`EnhancedFeatureRepository` instances are keyed by class, API host, and client -key. Keep per-user state isolated, and test with multiple users and clients when -changing cache keys, tracking, or sticky bucketing. The fixtures in -`tests/conftest.py` clean up async tasks and repository singletons between tests; -they do not replace testing the client's own cleanup. - -Use the existing `logging` loggers for diagnostics. Library code should not -configure the application's root logger or print to stdout. - -## Testing - -```bash -make test # full suite (pytest) -pytest tests/test_remote_eval.py # one file -pytest tests/test_growthbook.py::test_hash # one parametrized test -pytest -k sticky -x # matching tests, stop on first failure -make coverage # terminal report and HTML, opens a browser -``` - -For coverage without opening a browser: - -```bash -coverage run --source growthbook -m pytest -coverage report -m -coverage html -``` - -Tests use `pytest`, `pytest-mock`, and `pytest-asyncio`. Asyncio runs in strict -mode: mark async tests with `@pytest.mark.asyncio` and use -`@pytest_asyncio.fixture` for async fixtures. Mock external HTTP calls in unit -tests, and close clients and sessions created by the test. - -A bug fix should include a regression test that fails before the fix. For -generator changes, review `tests/codegen/expected_output.py` alongside the -generator and run both `tests/test_codegen.py` and `tests/test_typing.py`. - -### Conformance corpus - -`tests/cases.json` contains cases mirrored from the -[JS corpus](https://github.com/growthbook/growthbook/blob/main/packages/sdk-js/test/cases.json) -plus Python-local extensions and regressions. `tests/test_growthbook.py` loads -the corpus into parametrized tests. Investigate a failing shared case before -changing its expected result. - -CI compares the corpus with live JS `main` on every push: - -```bash -python tests/scripts/check_corpus_freshness.py -``` - -The command needs network access. To compare with a local JS checkout instead: - -```bash -python tests/scripts/check_corpus_freshness.py \ - --js-source /path/to/growthbook/packages/sdk-js/test/cases.json ``` -Missing cases and changed bodies of shared cases fail the check unless listed -in `tests/scripts/corpus_skiplist.json`. Python-only cases are reported but do -not fail it. Add a reason under `_reasons` for each intentional skip. The check -compares selected corpus sections; it does not replace running the tests. +Alternatively, use `uv venv --python 3.13 .venv` and `uv pip install` with the +same dependencies. Configure your editor to use `.venv/bin/python`. Unit tests +need no GrowthBook credentials or running server. -Preserve Python-local cases when syncing from upstream. For a parity bug that -affects other SDKs, propose a regression case in the JS corpus as well. +## Checks -### Benchmarks and integration checks - -Run benchmarks from the repository root with the virtual environment active: - -```bash -python tests/scripts/benchmark_eval_overhead.py 100000 -python tests/scripts/benchmark_async_client.py -python tests/scripts/benchmark_saved_groups.py -``` - -The first measures sync evaluation overhead; the second measures async -throughput, latency, and event-loop lag with simulated sticky bucket services. -The third measures ordinary and saved-group conditions through both clients; -use `--sdk-path /path/to/checkout --legacy-only` to compare existing workloads -before and after a change. None needs external services. Include before/after -results in the PR when -changing evaluation performance or concurrency behavior, using the same Python -version, machine, and workload for both runs. - -For remote-evaluation checks against an in-process test proxy: +With the virtual environment active, run from the repository root: ```bash -python tests/scripts/verify_remote_eval.py -``` - -This starts a localhost server. Real-proxy checks are separate and need Docker -and GrowthBook credentials; see [the verification scripts](tests/scripts/README.md) -for setup and teardown. - -## Code quality - -### Type checking - -MyPy is pinned in `requirements_dev.txt` and configured in `pyproject.toml`: - -```bash -make type-check -pytest tests/test_typing.py -q +pytest -q +mypy +flake8 . --extend-exclude=.venv --count --select=E9,F63,F7,F82 --show-source --statistics ``` -The typing regression suite checks valid usage and verifies that invalid calls -fail on the lines marked `# expect-error`. It always runs MyPy and also runs -Pyright when a `pyright` executable is on `PATH`. - -To reproduce CI's Pyright checks locally, install the version pinned in -`.github/workflows/main.yml`: +For typing changes, also run Pyright with the version pinned in CI: ```bash npm install -g pyright@1.1.403 @@ -229,93 +33,61 @@ pyright pytest tests/test_typing.py -q ``` -Keep the virtual environment active so Pyright can resolve the installed -dependencies. CI runs Pyright on Python 3.11; both checker configurations target -Python 3.9 language compatibility. - -### Linting and formatting - -CI's blocking lint check selects syntax and correctness errors. Exclude the -local virtual environment when running it from the repository root: - -```bash -flake8 . --extend-exclude=.venv --count --select=E9,F63,F7,F82 --show-source --statistics -``` +`make test` and `make type-check` wrap pytest and MyPy. `make coverage` produces +an HTML report. `make lint` uses older rules and fails on existing style issues; +the flake8 command above matches CI's blocking check. -CI also reports broader style warnings without failing the build: +## Working on the SDK -```bash -flake8 . --extend-exclude=.venv --count --exit-zero --max-complexity=25 --max-line-length=127 --statistics -``` +- `growthbook/core.py` contains evaluation shared by the sync `GrowthBook` + (`growthbook.py`) and async `GrowthBookClient` (`growthbook_client.py`). Test + both clients, including loading and refresh when payload handling changes. +- Keep library code compatible with Python 3.9 and add type annotations. + Public API changes should include typing regressions in `tests/typing/`. +- Mock external HTTP calls and close clients created in tests. Async tests use + `@pytest.mark.asyncio`; async fixtures use `@pytest_asyncio.fixture`. +- Follow the [JavaScript reference SDK](https://github.com/growthbook/growthbook/tree/main/packages/sdk-js) + for evaluation behavior. Preserve Python-local cases when syncing + `tests/cases.json` with the shared corpus. -The existing Make target is different: +For evaluation or corpus changes, check parity with JS `main` (requires network): ```bash -make lint +python tests/scripts/check_corpus_freshness.py ``` -It checks only `growthbook/growthbook.py`, uses a 150-character line limit, and -fails on existing style and unused-import issues. A failure there does not by -itself mean the environment is broken. Check whether a finding is introduced -by your change. - -There is no automatic formatter configured. Follow `.editorconfig` and the -surrounding code, use docstrings for declarations that need documentation, and -keep formatting changes limited to the code you touch. - -## Opening pull requests - -Open an issue first for API changes, new configuration, or behavior changes so -the design can be discussed. For a typo or an obvious fix, open a PR directly. - -1. Branch from `main` and keep the diff focused. -2. Add regression coverage and update `README.md` for changes to documented - behavior or usage. -3. Run `make test`, `make type-check`, and the blocking lint command above. - Run Pyright for typing changes, corpus freshness for evaluation or fixture - changes, and benchmarks for performance-sensitive changes. -4. Use descriptive commit subjects with conventional-commit prefixes such as - `fix:`, `feat:`, `docs:`, or `test:`. Release Please uses commit history to - prepare releases; make the eventual merge or squash subject describe the - change accurately. -5. Open the PR against `main`. Explain the problem, the resulting behavior, - and how you tested it. Call out compatibility and SDK parity implications. - -The build workflow (`.github/workflows/main.yml`) runs on pushes. It tests and -type-checks on Python 3.9–3.13, runs the two lint passes, checks Pyright and its -typing regressions on Python 3.11, and checks corpus freshness against JS -`main`. A local test pass on one interpreter does not cover the full matrix. +Use `--js-source /path/to/cases.json` for a local reference. Missing or changed +shared cases fail unless listed with a reason in `tests/scripts/corpus_skiplist.json`. -Push follow-up commits rather than force-pushing where possible so review -comments stay attached to the code they discuss. +For performance changes, compare both checkouts using the same interpreter and +machine. Benchmark commands and integration checks are in +[tests/scripts/README.md](tests/scripts/README.md). -## Releasing +## Pull requests -Maintainers handle releases through `.github/workflows/release-please.yml`: +Discuss public API changes in an issue first. Branch from `main`, include +regression tests and relevant usage docs, and run the checks above. Describe +behavior changes, compatibility concerns, and validation in the PR. -1. Pushes to `main` run Release Please, which prepares a release PR with version - and changelog updates. -2. Merging the release PR lets Release Please create the release and tag. -3. When a release is created, the publishing job runs tests on Python 3.11, - builds the source distribution and wheel with `python -m build`, and uploads - them to PyPI. +Use conventional commit subjects (`feat:`, `fix:`, etc.); Release Please uses +them to determine versions and changelog entries. Keep the merge or squash +subject consistent with the change. -`setup.py` reads the package version from `growthbook/__init__.py`. -`release-please-config.json` and `.release-please-manifest.json` configure and -track releases. Contributors normally do not need to edit version numbers or -`CHANGELOG.md`; include enough detail in the PR for a useful changelog entry. +## Releases -`make dist`, `make release`, and the bumpversion files are legacy tooling. -In particular, `make release` uploads directly to PyPI; it is not a local -verification command or the normal release process. +The [Release Please workflow](.github/workflows/release-please.yml) creates a +release PR after changes reach `main`. Merging that PR creates the tag and +GitHub release, then tests, builds, and publishes the package to PyPI. -## Getting help +`setup.py` reads the version from `growthbook/__init__.py`. Contributors normally +leave version and changelog updates to Release Please. The legacy `make release` +command uploads directly to PyPI; it is not a verification command. -- [Slack community](https://slack.growthbook.io?ref=contributing) -- [Open an issue](https://github.com/growthbook/growthbook-python/issues) -- [Python SDK docs](https://docs.growthbook.io/lib/python) +## Help -Found a security vulnerability? Email security@growthbook.io instead of filing -a public issue. See the [security policy](https://github.com/growthbook/growthbook/blob/main/SECURITY.md). +- [SDK docs](https://docs.growthbook.io/lib/python) +- [Issues](https://github.com/growthbook/growthbook-python/issues) +- [Slack](https://slack.growthbook.io?ref=contributing) -Contributors are expected to follow the [Code of Conduct](https://github.com/growthbook/growthbook/blob/main/CODE_OF_CONDUCT.md). +Report vulnerabilities to security@growthbook.io. Follow the +[Code of Conduct](https://github.com/growthbook/growthbook/blob/main/CODE_OF_CONDUCT.md). diff --git a/README.md b/README.md index 43bebf1..411b098 100644 --- a/README.md +++ b/README.md @@ -409,27 +409,15 @@ or typed entries: } ``` -A rule references either type with a top-level condition such as -`{"$savedGroup": {"id": "beta"}}`. For a list group, an optional `attributeKey` -in the reference overrides the group's attribute. Condition groups evaluate -against the current attributes and can reference other groups; cyclic -references return no match. References compose with `$and`, `$or`, `$nor`, -and `$not`, including in prerequisite conditions (which see the parent's -`value`, not the user's attributes). +A rule references either type with `{"$savedGroup": {"id": "beta"}}`. List +references can override `attributeKey`; condition groups can reference other +groups, with cycle detection. Legacy `$inGroup` and `$notInGroup` also accept +typed list entries. When loading a payload yourself, use `gb.set_payload(payload)` or -`await client.set_payload(payload)` so the saved groups accompany the features. -Legacy `$inGroup` and `$notInGroup` conditions accept both bare arrays and typed -list entries. A missing group still passes `$notInGroup`; a present malformed -entry, condition group, or unknown group type matches neither legacy operator. - -Receiving v2 payloads requires an SDK version registered with the capability -and an SDK Connection set to **All Saved Groups** under **Pass Saved Groups by -reference**. Before publication, a new version can be registered with -`prerelease: true` for testing by entering that version manually on the connection. -Remove the prerelease flag and regenerate `CAPABILITIES.md` when the version is -published. Remote-evaluation connections require the proxy's JS SDK dependency -to support v2 before the back end enables the capability for them. +`await client.set_payload(payload)` to load both features and saved groups. +Receiving v2 payloads requires a registered SDK version with this capability +and **All Saved Groups** under the SDK Connection's **Pass Saved Groups by reference**. ## GrowthBook class diff --git a/docs/benchmarks/saved-group-references-v2.jsonl b/docs/benchmarks/saved-group-references-v2.jsonl deleted file mode 100644 index 6579f7a..0000000 --- a/docs/benchmarks/saved-group-references-v2.jsonl +++ /dev/null @@ -1,157 +0,0 @@ -{"baseline": "e6e7773cfbe7aa00202708d3fa49bca849797602", "candidate": "75c1c1a1789dd53f55cdecf6dac23084ba8058f8", "core_sha256": "ee3481f6b31a83ef136cf7afe27e74fda010a46819b0204fe8641e9d00aec2fd", "date": "2026-09-25", "js_main": "e613b6149809c71f5155f6af6cd92a437b447c01", "kind": "metadata", "order": ["base", "final", "final", "base", "base", "final"], "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", "python": "3.13.15"} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.91525625, "min_us": 0.8912521, "python": "3.13.15", "samples_us": [0.8997062499999999, 0.93126665, 0.96367085, 0.8912521, 0.91284165, 0.91539585, 0.91525625], "scenario": "default", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 0.999075, "min_us": 0.97498125, "python": "3.13.15", "samples_us": [1.02040415, 1.0007854, 0.999075, 0.9828937499999999, 0.98577705, 0.97498125, 1.0121375000000001], "scenario": "default", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.706862500000001, "min_us": 4.6425458, "python": "3.13.15", "samples_us": [4.706862500000001, 4.8592896, 4.84730625, 4.7390979, 4.65432915, 4.6425458, 4.65135625], "scenario": "experiment", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.53742085, "min_us": 4.44055, "python": "3.13.15", "samples_us": [4.568325, 4.6216479, 4.53742085, 4.4692583500000005, 4.54255, 4.44055, 4.50575], "scenario": "experiment", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.52416875, "min_us": 1.48246875, "python": "3.13.15", "samples_us": [1.5231312499999998, 1.52603125, 1.5018270500000002, 1.48246875, 1.52416875, 1.54145625, 1.58462085], "scenario": "plain-condition", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.63176665, "min_us": 1.6008479, "python": "3.13.15", "samples_us": [1.61989375, 1.64097915, 1.6819791499999999, 1.63176665, 1.6156541500000001, 1.6008479, 1.64098125], "scenario": "plain-condition", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.7111229, "min_us": 2.6280187500000003, "python": "3.13.15", "samples_us": [2.74357915, 3.1726687499999997, 2.69193335, 2.6280187500000003, 2.6663875, 2.7111229, 2.7519583499999998], "scenario": "nested-condition", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.8125875000000002, "min_us": 2.7775396, "python": "3.13.15", "samples_us": [2.9263646, 2.7775396, 2.8125875000000002, 2.783575, 2.82102705, 2.83378955, 2.7801895500000002], "scenario": "nested-condition", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.70637705, "min_us": 1.63776875, "python": "3.13.15", "samples_us": [1.65034795, 1.63776875, 1.82824585, 1.7245583500000001, 1.76135835, 1.638075, 1.70637705], "scenario": "legacy-list-10", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7654521, "min_us": 1.75809795, "python": "3.13.15", "samples_us": [1.8243146, 1.75809795, 1.7654521, 1.77506875, 1.7876229000000001, 1.7623145999999998, 1.76419375], "scenario": "legacy-list-10", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.253514600000001, "min_us": 6.073370799999999, "python": "3.13.15", "samples_us": [6.30440625, 6.073370799999999, 6.333629200000001, 6.3787042000000005, 6.253514600000001, 6.129608350000001, 6.146525], "scenario": "legacy-list-1000", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.3326, "min_us": 6.2001958, "python": "3.13.15", "samples_us": [6.4191667, 6.357431249999999, 6.3326, 6.3213625, 6.2001958, 6.268572949999999, 6.410858350000001], "scenario": "legacy-list-1000", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.76250625, "min_us": 8.501020800000001, "python": "3.13.15", "samples_us": [8.76250625, 8.773064549999999, 8.9165521, 8.8681479, 8.57505625, 8.57900835, 8.501020800000001], "scenario": "legacy-array-1000", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.68685, "min_us": 8.4751042, "python": "3.13.15", "samples_us": [8.7025875, 8.4751042, 8.72375205, 8.68685, 8.48396045, 8.6664917, 9.10630415], "scenario": "legacy-array-1000", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6768604, "min_us": 1.6620479499999998, "python": "3.13.15", "samples_us": [1.6684583, 1.67775, 1.7091729, 1.6620479499999998, 1.68225625, 1.6768604, 1.67084375], "scenario": "legacy-not-in-group", "trial": 1} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.8142521, "min_us": 1.71752915, "python": "3.13.15", "samples_us": [1.83485415, 1.8142521, 1.8146875, 1.8506500000000001, 1.8081958500000002, 1.76027295, 1.71752915], "scenario": "legacy-not-in-group", "trial": 1} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.91474585, "min_us": 0.88725835, "python": "3.13.15", "samples_us": [0.9193875, 0.93439585, 0.91474585, 0.9085583500000001, 0.97683545, 0.9123354, 0.88725835], "scenario": "default", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 0.99360205, "min_us": 0.9698125, "python": "3.13.15", "samples_us": [0.9785458499999999, 0.9728312499999999, 1.0278146, 1.00254795, 0.99360205, 1.0011146, 0.9698125], "scenario": "default", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.6533521, "min_us": 4.38875835, "python": "3.13.15", "samples_us": [4.443745799999999, 4.38875835, 4.6533521, 4.6271375, 4.7675417, 4.67882295, 4.751027049999999], "scenario": "experiment", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.51281875, "min_us": 4.4165354, "python": "3.13.15", "samples_us": [4.74438125, 4.5975917, 4.56803125, 4.51281875, 4.44649795, 4.42695, 4.4165354], "scenario": "experiment", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.5481395500000001, "min_us": 1.4880125, "python": "3.13.15", "samples_us": [1.61680205, 1.5054146000000002, 1.57598335, 1.4880125, 1.5561125, 1.5481395500000001, 1.5113583000000002], "scenario": "plain-condition", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.5771729499999998, "min_us": 1.5627333, "python": "3.13.15", "samples_us": [1.57369165, 1.5771729499999998, 1.58562915, 1.5627333, 1.56916455, 1.6123832999999999, 1.5776396], "scenario": "plain-condition", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.69394165, "min_us": 2.60333125, "python": "3.13.15", "samples_us": [2.8280666500000002, 2.6632792, 2.69394165, 2.66217915, 2.7809479, 2.7127146, 2.60333125], "scenario": "nested-condition", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.7569979, "min_us": 2.6780999999999997, "python": "3.13.15", "samples_us": [3.2351604000000003, 3.75744165, 2.7474021, 2.7569979, 2.7303854500000004, 2.6780999999999997, 2.7830792], "scenario": "nested-condition", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6455687499999998, "min_us": 1.6046542, "python": "3.13.15", "samples_us": [1.6121646, 1.6222895499999999, 1.67638335, 1.68984795, 1.6767896, 1.6046542, 1.6455687499999998], "scenario": "legacy-list-10", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7290854, "min_us": 1.7090375, "python": "3.13.15", "samples_us": [1.7236270999999999, 1.7090375, 1.8340604, 1.74840625, 1.81980625, 1.7290854, 1.71015], "scenario": "legacy-list-10", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.2734542, "min_us": 6.0787792, "python": "3.13.15", "samples_us": [7.5588, 6.10987295, 6.0787792, 6.299487500000001, 6.200808299999999, 6.31490835, 6.2734542], "scenario": "legacy-list-1000", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.122327100000001, "min_us": 6.03718335, "python": "3.13.15", "samples_us": [6.1190708, 6.122327100000001, 6.2761729, 6.03718335, 6.0997625, 6.15894165, 6.3735520999999995], "scenario": "legacy-list-1000", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.4603375, "min_us": 8.2003542, "python": "3.13.15", "samples_us": [8.25294585, 8.25831875, 8.54996665, 8.645237499999999, 8.4603375, 8.5577625, 8.2003542], "scenario": "legacy-array-1000", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.250675, "min_us": 8.1360167, "python": "3.13.15", "samples_us": [8.345635399999999, 8.18569795, 8.1360167, 8.231854149999998, 8.47146665, 8.3660583, 8.250675], "scenario": "legacy-array-1000", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6148104, "min_us": 1.58423955, "python": "3.13.15", "samples_us": [1.66056665, 1.6008958500000001, 1.58423955, 1.6127625, 1.64599585, 1.62270625, 1.6148104], "scenario": "legacy-not-in-group", "trial": 2} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.76300835, "min_us": 1.73237915, "python": "3.13.15", "samples_us": [1.73237915, 1.76300835, 1.8029604, 1.7755562500000002, 1.75437295, 1.7917292, 1.7468458], "scenario": "legacy-not-in-group", "trial": 2} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.89704375, "min_us": 0.8823917, "python": "3.13.15", "samples_us": [0.8959520999999999, 0.89704375, 0.89418335, 0.8823917, 0.90855625, 0.89803125, 0.9011667], "scenario": "default", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.0110646, "min_us": 0.9736479, "python": "3.13.15", "samples_us": [0.9736479, 0.98921875, 1.01546665, 1.0148728999999999, 1.0110646, 0.99019585, 1.0207875], "scenario": "default", "trial": 3} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.41017085, "min_us": 4.360264600000001, "python": "3.13.15", "samples_us": [4.360264600000001, 4.4131479, 4.4001417, 4.38338125, 4.41017085, 4.4365353999999995, 4.52211455], "scenario": "experiment", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.613125, "min_us": 4.3757771000000005, "python": "3.13.15", "samples_us": [4.461525, 4.3757771000000005, 4.6635145499999995, 4.57378125, 4.6531604, 4.62615625, 4.613125], "scenario": "experiment", "trial": 3} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.5349104, "min_us": 1.4938604, "python": "3.13.15", "samples_us": [1.5166021, 1.5349104, 1.5509104, 1.5962417, 1.55605835, 1.4938604, 1.5266229], "scenario": "plain-condition", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.5997146, "min_us": 1.5532521, "python": "3.13.15", "samples_us": [1.6251167, 1.62797705, 1.5914437499999998, 1.5997146, 1.63396455, 1.595675, 1.5532521], "scenario": "plain-condition", "trial": 3} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.6744875, "min_us": 2.5694, "python": "3.13.15", "samples_us": [2.6127458499999996, 2.6744875, 2.70914585, 2.63179375, 2.67840625, 2.5694, 2.7041853999999996], "scenario": "nested-condition", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.73277705, "min_us": 2.66449375, "python": "3.13.15", "samples_us": [2.74719375, 2.7775542, 2.69385, 2.66449375, 2.73277705, 2.6922833500000003, 2.82133125], "scenario": "nested-condition", "trial": 3} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.61557915, "min_us": 1.5849208000000001, "python": "3.13.15", "samples_us": [1.61557915, 1.5978021, 1.6208125, 1.60007915, 1.6609583, 1.5849208000000001, 1.67297085], "scenario": "legacy-list-10", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7643583, "min_us": 1.6932271, "python": "3.13.15", "samples_us": [1.70834795, 1.6932271, 1.7643583, 1.801025, 1.7724916499999999, 1.7626125, 1.8133770999999999], "scenario": "legacy-list-10", "trial": 3} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.3564853999999995, "min_us": 6.20002085, "python": "3.13.15", "samples_us": [6.47531045, 6.4059875, 6.3106104, 6.3564853999999995, 6.4675083, 6.2191292, 6.20002085], "scenario": "legacy-list-1000", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.22628335, "min_us": 6.12835, "python": "3.13.15", "samples_us": [6.1662271, 6.2325625, 6.248925, 6.12835, 6.20111665, 6.22628335, 6.272925], "scenario": "legacy-list-1000", "trial": 3} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.7573875, "min_us": 8.64815415, "python": "3.13.15", "samples_us": [8.7573875, 8.64815415, 8.7726521, 8.726835450000001, 9.1262229, 8.77975, 8.73453125], "scenario": "legacy-array-1000", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.811931249999999, "min_us": 8.7077708, "python": "3.13.15", "samples_us": [8.7684625, 9.171306249999999, 8.87372085, 8.7077708, 8.77232915, 9.0740625, 8.811931249999999], "scenario": "legacy-array-1000", "trial": 3} -{"checkout": "base", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.55896875, "min_us": 1.5382271, "python": "3.13.15", "samples_us": [1.5382271, 1.5446875, 1.5879583, 1.5518771, 2.12846875, 1.8725771, 1.55896875], "scenario": "legacy-not-in-group", "trial": 3} -{"checkout": "base", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7298145999999999, "min_us": 1.71821665, "python": "3.13.15", "samples_us": [1.86085, 1.89499795, 1.7265708, 1.7415604, 1.7298145999999999, 1.71821665, 1.7265229500000001], "scenario": "legacy-not-in-group", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.9501854, "min_us": 0.9168875000000001, "python": "3.13.15", "samples_us": [0.9369895500000001, 0.95731665, 0.9501854, 0.9403937499999999, 1.04558125, 1.0520895499999998, 0.9168875000000001], "scenario": "default", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.0163083, "min_us": 1.00188335, "python": "3.13.15", "samples_us": [1.0407875, 1.0366292000000001, 1.0113792, 1.0163083, 1.1125604, 1.00188335, 1.0048771], "scenario": "default", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.5898417, "min_us": 4.4328083, "python": "3.13.15", "samples_us": [4.4328083, 4.52304375, 5.08686665, 4.585875, 4.5898417, 4.7070667, 4.8008021], "scenario": "experiment", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.6828708500000005, "min_us": 4.57466665, "python": "3.13.15", "samples_us": [4.74911875, 4.6135292, 4.57466665, 4.6828708500000005, 4.71356455, 4.7073395499999995, 4.60502915], "scenario": "experiment", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.5812728999999999, "min_us": 1.5340625, "python": "3.13.15", "samples_us": [1.5812728999999999, 1.5340625, 1.6008521, 1.61313125, 1.5955896, 1.55369585, 1.5746875], "scenario": "plain-condition", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.61463335, "min_us": 1.60075415, "python": "3.13.15", "samples_us": [1.65883335, 1.61463335, 1.60075415, 1.61993125, 1.6063646, 1.6028396, 1.62542085], "scenario": "plain-condition", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.78591875, "min_us": 2.72218125, "python": "3.13.15", "samples_us": [2.86320415, 2.78591875, 2.76523125, 2.7335021, 2.72218125, 2.78785205, 2.79923335], "scenario": "nested-condition", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.9130229, "min_us": 2.8123395500000004, "python": "3.13.15", "samples_us": [2.94036045, 2.9741999999999997, 2.9130229, 2.83405205, 2.8909499999999997, 2.9769396, 2.8123395500000004], "scenario": "nested-condition", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.734125, "min_us": 1.691375, "python": "3.13.15", "samples_us": [1.6941708000000002, 1.691375, 1.734125, 1.7142125000000001, 1.78829165, 1.7566041499999998, 1.82087085], "scenario": "legacy-list-10", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.89686875, "min_us": 1.8760541499999999, "python": "3.13.15", "samples_us": [1.89070625, 1.96609795, 1.9297396, 1.905875, 1.8906229, 1.8760541499999999, 1.89686875], "scenario": "legacy-list-10", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6565375, "min_us": 1.64021875, "python": "3.13.15", "samples_us": [1.6439896, 1.64021875, 1.665075, 1.6427583, 1.6859604, 1.6565375, 1.6599146], "scenario": "v2-list-10", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.68880625, "min_us": 1.6711395500000001, "python": "3.13.15", "samples_us": [1.6783458, 1.68416045, 2.1187332999999997, 1.73556045, 1.77300625, 1.68880625, 1.6711395500000001], "scenario": "v2-list-10", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.19900625, "min_us": 6.16515, "python": "3.13.15", "samples_us": [6.16515, 6.19550835, 6.1751854, 6.19900625, 6.20026875, 6.3707875000000005, 6.2020979], "scenario": "legacy-list-1000", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.29534375, "min_us": 6.2655125, "python": "3.13.15", "samples_us": [6.29534375, 6.39862085, 6.4570958, 6.3455979, 6.2655125, 6.266958300000001, 6.2697978999999995], "scenario": "legacy-list-1000", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.04683125, "min_us": 5.91361665, "python": "3.13.15", "samples_us": [6.045649999999999, 6.03578335, 6.16063335, 6.04683125, 6.1772124999999996, 6.135493749999999, 5.91361665], "scenario": "v2-list-1000", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.24695415, "min_us": 6.125920850000001, "python": "3.13.15", "samples_us": [6.14290835, 6.28399375, 6.228868749999999, 6.3402083000000005, 7.2605333, 6.125920850000001, 6.24695415], "scenario": "v2-list-1000", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 9.22092915, "min_us": 8.8211271, "python": "3.13.15", "samples_us": [8.963700000000001, 8.8211271, 9.22092915, 9.45024165, 9.253768749999999, 9.2312396, 8.9192104], "scenario": "legacy-array-1000", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.89325835, "min_us": 8.77605835, "python": "3.13.15", "samples_us": [8.80182915, 9.016825, 8.89325835, 8.77605835, 8.852118749999999, 8.964775, 8.99563545], "scenario": "legacy-array-1000", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.73110835, "min_us": 1.70992295, "python": "3.13.15", "samples_us": [1.73110835, 1.84421875, 1.9713270999999999, 1.7134729, 1.8164854, 1.7197770499999998, 1.70992295], "scenario": "legacy-not-in-group", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.8336292, "min_us": 1.7653333, "python": "3.13.15", "samples_us": [1.81821875, 1.8336292, 1.8361958, 1.85109375, 1.8723937499999999, 1.7653333, 1.8048333], "scenario": "legacy-not-in-group", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.82614375, "min_us": 1.75386875, "python": "3.13.15", "samples_us": [1.86775, 1.8218417, 1.8886979, 1.84129795, 1.82614375, 1.7737271000000001, 1.75386875], "scenario": "v2-condition", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.89778125, "min_us": 1.8499375, "python": "3.13.15", "samples_us": [1.8607979499999998, 2.0668625, 1.89778125, 1.86859375, 1.9527750000000001, 1.8499375, 1.9346312499999998], "scenario": "v2-condition", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 3.0239895999999997, "min_us": 2.95849375, "python": "3.13.15", "samples_us": [3.0437354, 2.95849375, 3.1286583500000003, 3.0239895999999997, 3.0745603999999997, 2.9833333499999997, 2.96141875], "scenario": "v2-nested-condition", "trial": 1} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 3.0929708000000002, "min_us": 3.0620666500000002, "python": "3.13.15", "samples_us": [3.1599625000000002, 3.1578542, 3.1047125, 3.0620666500000002, 3.07316455, 3.0859978999999997, 3.0929708000000002], "scenario": "v2-nested-condition", "trial": 1} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.95605, "min_us": 0.9466687500000001, "python": "3.13.15", "samples_us": [0.95022085, 0.9466687500000001, 0.97379165, 0.98290625, 0.9506042, 0.95605, 0.9765979], "scenario": "default", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.0587208499999998, "min_us": 1.0116083, "python": "3.13.15", "samples_us": [1.0116083, 1.03121875, 1.0647375, 1.06830625, 1.0706396, 1.0587208499999998, 1.03341875], "scenario": "default", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.6418187500000005, "min_us": 4.5037291999999995, "python": "3.13.15", "samples_us": [4.78569375, 4.7059896, 4.6778249999999995, 4.6418187500000005, 4.56280415, 4.57425, 4.5037291999999995], "scenario": "experiment", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 5.51788125, "min_us": 4.52385625, "python": "3.13.15", "samples_us": [4.52385625, 4.603345849999999, 4.632860399999999, 5.93967295, 6.853202100000001, 14.9319854, 5.51788125], "scenario": "experiment", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6151396, "min_us": 1.58689165, "python": "3.13.15", "samples_us": [1.58689165, 1.6851729, 1.6809979499999999, 1.6151396, 1.6202479, 1.59508125, 1.60548125], "scenario": "plain-condition", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7059396, "min_us": 1.6810916500000002, "python": "3.13.15", "samples_us": [1.7028125, 1.6810916500000002, 1.71805835, 1.6813271, 1.7059396, 1.72374165, 1.91619375], "scenario": "plain-condition", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.78348125, "min_us": 2.71338545, "python": "3.13.15", "samples_us": [3.03188125, 2.79590205, 2.72081875, 2.71338545, 2.78348125, 2.75510415, 2.81181875], "scenario": "nested-condition", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.8565125, "min_us": 2.78348125, "python": "3.13.15", "samples_us": [2.8702104, 2.8565125, 2.8226457999999996, 2.79165, 3.1992875, 2.78348125, 2.9433312500000004], "scenario": "nested-condition", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.67728335, "min_us": 1.6441375, "python": "3.13.15", "samples_us": [1.6441375, 1.7104229499999999, 1.71604585, 1.67728335, 1.6633978999999999, 1.70024375, 1.6464375], "scenario": "legacy-list-10", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.80724165, "min_us": 1.7681667, "python": "3.13.15", "samples_us": [1.77194795, 1.7681667, 1.8172292, 1.80724165, 1.8394458, 1.80647295, 1.8273167000000001], "scenario": "legacy-list-10", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.55355, "min_us": 1.5313208, "python": "3.13.15", "samples_us": [1.55355, 1.5453875000000001, 1.6414021, 1.54465625, 1.5939396, 1.56821665, 1.5313208], "scenario": "v2-list-10", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.6804229, "min_us": 1.63771875, "python": "3.13.15", "samples_us": [1.6982562499999998, 1.6722271, 1.72202085, 1.63771875, 1.6804229, 1.73258125, 1.6642021], "scenario": "v2-list-10", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.17611875, "min_us": 6.0599729, "python": "3.13.15", "samples_us": [6.0599729, 6.11313335, 6.35407915, 6.17611875, 6.2387833, 6.1717125, 6.1874771], "scenario": "legacy-list-1000", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.23740835, "min_us": 6.1702104, "python": "3.13.15", "samples_us": [6.23740835, 6.17885415, 6.29840205, 6.1702104, 6.190608350000001, 6.260325, 6.32633125], "scenario": "legacy-list-1000", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.0269271, "min_us": 5.941304199999999, "python": "3.13.15", "samples_us": [6.0269271, 6.0803896, 6.0508500000000005, 5.941304199999999, 5.9605042, 5.94300415, 6.0920917], "scenario": "v2-list-1000", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.25116665, "min_us": 6.0390229, "python": "3.13.15", "samples_us": [6.25116665, 6.16399795, 6.3913604500000005, 6.37139375, 6.3981396, 6.0390229, 6.176960449999999], "scenario": "v2-list-1000", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.4854146, "min_us": 8.3594167, "python": "3.13.15", "samples_us": [8.51044585, 8.60761875, 8.506662500000001, 8.39770835, 8.36065415, 8.3594167, 8.4854146], "scenario": "legacy-array-1000", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.772197949999999, "min_us": 8.60236665, "python": "3.13.15", "samples_us": [8.772197949999999, 8.85310415, 8.7743229, 8.63303335, 8.693689549999998, 8.60236665, 9.15104165], "scenario": "legacy-array-1000", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6505958, "min_us": 1.57345625, "python": "3.13.15", "samples_us": [1.59058955, 1.63530835, 1.68686045, 1.6505958, 1.6950875, 1.6640875000000002, 1.57345625], "scenario": "legacy-not-in-group", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.74729165, "min_us": 1.66358125, "python": "3.13.15", "samples_us": [1.74729165, 1.7421854, 1.7461146, 1.66358125, 1.8213521000000001, 1.7739604, 1.7948521000000002], "scenario": "legacy-not-in-group", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.7503604, "min_us": 1.7275125, "python": "3.13.15", "samples_us": [1.72860835, 1.7714520500000002, 1.7415916999999999, 1.7275125, 1.76431875, 1.76111875, 1.7503604], "scenario": "v2-condition", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.8784792, "min_us": 1.8351646000000001, "python": "3.13.15", "samples_us": [1.8351646000000001, 1.8472457999999998, 1.8446937499999998, 1.91453125, 1.9891937499999999, 1.9292083, 1.8784792], "scenario": "v2-condition", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.9533604, "min_us": 2.9218312500000003, "python": "3.13.15", "samples_us": [2.9513896, 2.9304417, 2.9751792, 2.9218312500000003, 2.9664708500000003, 3.03365, 2.9533604], "scenario": "v2-nested-condition", "trial": 2} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 3.0615124999999996, "min_us": 3.020025, "python": "3.13.15", "samples_us": [3.2403625000000003, 3.0282583, 3.0478521, 3.0615124999999996, 3.0631875, 3.14950625, 3.020025], "scenario": "v2-nested-condition", "trial": 2} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 0.8945125, "min_us": 0.88124375, "python": "3.13.15", "samples_us": [0.93894585, 0.8945125, 0.8949771, 0.88124375, 0.9173666500000001, 0.8933917, 0.88315], "scenario": "default", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 0.9990229500000001, "min_us": 0.9580187499999999, "python": "3.13.15", "samples_us": [0.96249585, 0.9580187499999999, 0.9990229500000001, 1.01775415, 1.0010375, 1.00807915, 0.9884562499999999], "scenario": "default", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 4.43501875, "min_us": 4.4065458500000005, "python": "3.13.15", "samples_us": [4.41203125, 4.4065458500000005, 4.51653335, 4.42620625, 4.46205625, 4.45600205, 4.43501875], "scenario": "experiment", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 4.41778335, "min_us": 4.3800021, "python": "3.13.15", "samples_us": [4.41778335, 4.4563729499999996, 4.4309229, 4.3800021, 4.4289, 4.41570835, 4.4085125], "scenario": "experiment", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.51005, "min_us": 1.489, "python": "3.13.15", "samples_us": [1.489, 1.5491041499999998, 1.5155146, 1.51005, 1.4986062500000001, 1.4917208499999999, 1.5106791499999999], "scenario": "plain-condition", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.6117249999999999, "min_us": 1.5906229, "python": "3.13.15", "samples_us": [1.5906229, 1.5932875, 1.59845625, 1.6222771, 1.6117249999999999, 1.61611455, 1.65730625], "scenario": "plain-condition", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.67414585, "min_us": 2.64751875, "python": "3.13.15", "samples_us": [2.6580812500000004, 2.71510835, 2.6937271000000003, 2.66913125, 2.747275, 2.67414585, 2.64751875], "scenario": "nested-condition", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 2.7658354, "min_us": 2.7532604500000004, "python": "3.13.15", "samples_us": [2.7532604500000004, 2.76299375, 2.78788125, 2.7806937499999997, 2.7658354, 2.7633833, 2.7867646], "scenario": "nested-condition", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.70064795, "min_us": 1.68446875, "python": "3.13.15", "samples_us": [1.70064795, 1.7005875000000001, 1.68446875, 1.8260416499999998, 1.7762833, 1.7086854, 1.68660625], "scenario": "legacy-list-10", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.79745625, "min_us": 1.7823646000000002, "python": "3.13.15", "samples_us": [1.97803125, 2.0591354, 1.7883896, 1.78519375, 1.79745625, 1.83838335, 1.7823646000000002], "scenario": "legacy-list-10", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.56695, "min_us": 1.5335999999999999, "python": "3.13.15", "samples_us": [1.5658687500000001, 1.6552979, 1.60516875, 1.53732085, 1.5938687500000002, 1.56695, 1.5335999999999999], "scenario": "v2-list-10", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.6390021, "min_us": 1.62625, "python": "3.13.15", "samples_us": [1.6390021, 1.6351875, 1.65627705, 1.62625, 1.7159271, 1.6360208500000002, 1.64445835], "scenario": "v2-list-10", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.34747915, "min_us": 6.14897915, "python": "3.13.15", "samples_us": [6.22051875, 6.4223646, 6.34747915, 6.29253125, 6.3692083, 6.4708771, 6.14897915], "scenario": "legacy-list-1000", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.3347375, "min_us": 6.18475415, "python": "3.13.15", "samples_us": [6.3370271, 6.3347375, 6.20884375, 6.18475415, 6.39333125, 6.374335449999999, 6.2573604], "scenario": "legacy-list-1000", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 6.1611396, "min_us": 5.86745835, "python": "3.13.15", "samples_us": [6.2267792, 5.86745835, 6.1510875, 6.332908349999999, 6.18880415, 6.1611396, 6.0496875], "scenario": "v2-list-1000", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 6.3854229, "min_us": 6.2034708499999995, "python": "3.13.15", "samples_us": [6.35958335, 7.559699999999999, 6.634412500000001, 6.614266649999999, 6.3854229, 6.2174521, 6.2034708499999995], "scenario": "v2-list-1000", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 8.3610521, "min_us": 8.24090835, "python": "3.13.15", "samples_us": [8.617368749999999, 8.338177100000001, 8.33557915, 8.3610521, 8.381060450000001, 8.24090835, 8.5070125], "scenario": "legacy-array-1000", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 8.52853545, "min_us": 8.3322021, "python": "3.13.15", "samples_us": [8.59360625, 8.4152375, 8.52304585, 8.661845849999999, 8.52853545, 8.723395850000001, 8.3322021], "scenario": "legacy-array-1000", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.6642875, "min_us": 1.64457915, "python": "3.13.15", "samples_us": [1.6627979499999999, 1.7190208500000002, 1.71021875, 1.6642875, 1.6580082999999999, 1.64457915, 1.6694979], "scenario": "legacy-not-in-group", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.7586104, "min_us": 1.7374625000000001, "python": "3.13.15", "samples_us": [1.823375, 1.7524771000000001, 1.7588062500000001, 1.76736045, 1.74952915, 1.7374625000000001, 1.7586104], "scenario": "legacy-not-in-group", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 1.72567705, "min_us": 1.70137085, "python": "3.13.15", "samples_us": [1.7205979500000002, 1.72473335, 1.7927354, 1.8617854500000002, 1.7502875, 1.70137085, 1.72567705], "scenario": "v2-condition", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 1.86705415, "min_us": 1.82103125, "python": "3.13.15", "samples_us": [1.87314165, 1.82103125, 1.8302104, 1.86801665, 1.8680750000000002, 1.86705415, 1.85595625], "scenario": "v2-condition", "trial": 3} -{"checkout": "final", "client": "sync", "iterations": 20000, "kind": "targeted", "median_us": 2.99215, "min_us": 2.88095205, "python": "3.13.15", "samples_us": [2.88095205, 2.88198335, 2.99215, 3.0952542, 3.07332915, 2.9766624999999998, 3.0010583499999997], "scenario": "v2-nested-condition", "trial": 3} -{"checkout": "final", "client": "async", "iterations": 20000, "kind": "targeted", "median_us": 3.1180896, "min_us": 3.0551666500000003, "python": "3.13.15", "samples_us": [3.08345415, 3.1180896, 3.14001455, 3.09824375, 3.1351999999999998, 3.1388625, 3.0551666500000003], "scenario": "v2-nested-condition", "trial": 3} -{"checkout": "base", "kind": "existing_sync", "scenario": "default-value", "trial": 1, "us": 0.93} -{"checkout": "base", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 1, "us": 4.166} -{"checkout": "base", "kind": "existing_sync", "scenario": "default-value", "trial": 2, "us": 0.917} -{"checkout": "base", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 2, "us": 4.146} -{"checkout": "base", "kind": "existing_sync", "scenario": "default-value", "trial": 3, "us": 0.937} -{"checkout": "base", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 3, "us": 4.096} -{"checkout": "final", "kind": "existing_sync", "scenario": "default-value", "trial": 1, "us": 0.915} -{"checkout": "final", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 1, "us": 4.145} -{"checkout": "final", "kind": "existing_sync", "scenario": "default-value", "trial": 2, "us": 0.911} -{"checkout": "final", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 2, "us": 4.15} -{"checkout": "final", "kind": "existing_sync", "scenario": "default-value", "trial": 3, "us": 0.904} -{"checkout": "final", "kind": "existing_sync", "scenario": "experiment-rule", "trial": 3, "us": 4.081} -{"checkout": "base", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 5.26, "loop_lag_mean_ms": 5.263, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 80468.9, "trial": 1} -{"checkout": "base", "eval_p50_ms": 17.48, "eval_p95_ms": 18.51, "eval_p99_ms": 19.18, "kind": "existing_async", "loop_lag_max_ms": 0.85, "loop_lag_mean_ms": 0.189, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 5560.6, "trial": 1} -{"checkout": "base", "eval_p50_ms": 3.54, "eval_p95_ms": 4.41, "eval_p99_ms": 4.68, "kind": "existing_async", "loop_lag_max_ms": 1.48, "loop_lag_mean_ms": 0.705, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 24549.1, "trial": 1} -{"checkout": "base", "eval_p50_ms": 2.42, "eval_p95_ms": 2.78, "eval_p99_ms": 2.83, "kind": "existing_async", "loop_lag_max_ms": 0.93, "loop_lag_mean_ms": 0.662, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 33028.7, "trial": 1} -{"checkout": "base", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.51, "loop_lag_mean_ms": 3.513, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 95458.9, "trial": 2} -{"checkout": "base", "eval_p50_ms": 20.53, "eval_p95_ms": 22.06, "eval_p99_ms": 22.74, "kind": "existing_async", "loop_lag_max_ms": 0.67, "loop_lag_mean_ms": 0.204, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4736.3, "trial": 2} -{"checkout": "base", "eval_p50_ms": 3.44, "eval_p95_ms": 3.93, "eval_p99_ms": 4.1, "kind": "existing_async", "loop_lag_max_ms": 1.26, "loop_lag_mean_ms": 0.77, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26321.0, "trial": 2} -{"checkout": "base", "eval_p50_ms": 2.09, "eval_p95_ms": 2.92, "eval_p99_ms": 3.03, "kind": "existing_async", "loop_lag_max_ms": 1.25, "loop_lag_mean_ms": 0.677, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 37538.6, "trial": 2} -{"checkout": "base", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.5, "loop_lag_mean_ms": 3.504, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 95568.4, "trial": 3} -{"checkout": "base", "eval_p50_ms": 19.87, "eval_p95_ms": 21.62, "eval_p99_ms": 22.34, "kind": "existing_async", "loop_lag_max_ms": 0.76, "loop_lag_mean_ms": 0.151, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4923.1, "trial": 3} -{"checkout": "base", "eval_p50_ms": 3.39, "eval_p95_ms": 3.77, "eval_p99_ms": 3.91, "kind": "existing_async", "loop_lag_max_ms": 1.15, "loop_lag_mean_ms": 0.82, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26758.0, "trial": 3} -{"checkout": "base", "eval_p50_ms": 2.06, "eval_p95_ms": 2.72, "eval_p99_ms": 3.14, "kind": "existing_async", "loop_lag_max_ms": 1.18, "loop_lag_mean_ms": 0.925, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 37858.2, "trial": 3} -{"checkout": "final", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.45, "loop_lag_mean_ms": 3.447, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 95855.1, "trial": 1} -{"checkout": "final", "eval_p50_ms": 20.02, "eval_p95_ms": 21.84, "eval_p99_ms": 22.64, "kind": "existing_async", "loop_lag_max_ms": 1.04, "loop_lag_mean_ms": 0.201, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4813.8, "trial": 1} -{"checkout": "final", "eval_p50_ms": 3.59, "eval_p95_ms": 4.15, "eval_p99_ms": 4.36, "kind": "existing_async", "loop_lag_max_ms": 1.36, "loop_lag_mean_ms": 0.618, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 25006.4, "trial": 1} -{"checkout": "final", "eval_p50_ms": 2.45, "eval_p95_ms": 2.84, "eval_p99_ms": 2.95, "kind": "existing_async", "loop_lag_max_ms": 1.09, "loop_lag_mean_ms": 0.774, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 32379.6, "trial": 1} -{"checkout": "final", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.01, "kind": "existing_async", "loop_lag_max_ms": 3.35, "loop_lag_mean_ms": 3.354, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 96738.7, "trial": 2} -{"checkout": "final", "eval_p50_ms": 20.56, "eval_p95_ms": 22.05, "eval_p99_ms": 23.08, "kind": "existing_async", "loop_lag_max_ms": 1.14, "loop_lag_mean_ms": 0.255, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4745.5, "trial": 2} -{"checkout": "final", "eval_p50_ms": 3.4, "eval_p95_ms": 3.83, "eval_p99_ms": 3.99, "kind": "existing_async", "loop_lag_max_ms": 1.09, "loop_lag_mean_ms": 0.749, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26598.5, "trial": 2} -{"checkout": "final", "eval_p50_ms": 2.12, "eval_p95_ms": 3.11, "eval_p99_ms": 3.18, "kind": "existing_async", "loop_lag_max_ms": 1.26, "loop_lag_mean_ms": 0.765, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 36426.1, "trial": 2} -{"checkout": "final", "eval_p50_ms": 0.01, "eval_p95_ms": 0.01, "eval_p99_ms": 0.02, "kind": "existing_async", "loop_lag_max_ms": 4.0, "loop_lag_mean_ms": 4.001, "scenario": "no-sticky-service", "sticky_docs_persisted": null, "throughput_rps": 89130.2, "trial": 3} -{"checkout": "final", "eval_p50_ms": 19.65, "eval_p95_ms": 21.25, "eval_p99_ms": 22.48, "kind": "existing_async", "loop_lag_max_ms": 1.21, "loop_lag_mean_ms": 0.198, "scenario": "sync-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 4932.5, "trial": 3} -{"checkout": "final", "eval_p50_ms": 3.41, "eval_p95_ms": 4.12, "eval_p99_ms": 4.49, "kind": "existing_async", "loop_lag_max_ms": 1.08, "loop_lag_mean_ms": 0.581, "scenario": "async-sticky-service", "sticky_docs_persisted": 1000, "throughput_rps": 26272.2, "trial": 3} -{"checkout": "final", "eval_p50_ms": 2.06, "eval_p95_ms": 2.58, "eval_p99_ms": 2.67, "kind": "existing_async", "loop_lag_max_ms": 1.22, "loop_lag_mean_ms": 0.818, "scenario": "async-hot-user", "sticky_docs_persisted": 1, "throughput_rps": 38236.0, "trial": 3} diff --git a/docs/benchmarks/saved-group-references-v2.md b/docs/benchmarks/saved-group-references-v2.md index b58dd09..0030dcd 100644 --- a/docs/benchmarks/saved-group-references-v2.md +++ b/docs/benchmarks/saved-group-references-v2.md @@ -1,119 +1,42 @@ -# Saved group references v2 verification +# Saved group references v2 benchmarks -Measured on 2026-09-25. No material performance regression was detected in the -tested existing workloads after deferring cycle-set allocation and keeping legacy -arrays on a direct lookup path. This is a local comparison, not proof of zero -regression on every workload or interpreter. +Measured 2026-09-25 on Python 3.13.15, macOS 26.6.2 ARM64: -## Reference and correctness - -- Implementation prompt: [Notion](https://app.notion.com/p/3dda857e74a0808499d6cfba00a18196). -- Merged reference: [growthbook#6942](https://github.com/growthbook/growthbook/pull/6942). -- Checked monorepo `main` at `e613b6149809c71f5155f6af6cd92a437b447c01`. -- `mongrule.ts`, `core.ts`, and the JS corpus are identical to reference commit - `023c21040b8878650f0e2cf0eb7ca44ed4b6d2f1`, which the Python port used. -- Corpus comparison: spec 0.9.0, no missing or drifted cases; 41 Python-local extras. -- Both clients run all feature and experiment v2 cases, including prerequisites. - Additional tests cover plaintext/encrypted payload injection and fetching, - refreshed membership, malformed groups, and concurrent async evaluations. -- Final verification: 1,154 tests passed, including the MyPy and Pyright typing - regressions. Standalone MyPy, Pyright, and CI's blocking lint check also pass. - Pytest reports one existing `event_loop_policy` deprecation warning. - -The updated prompt permits capability registration before publication only with -`prerelease: true` on a new version. Remove that flag and regenerate -`CAPABILITIES.md` when publishing. Python has no such entry yet. Remote evaluation -depends on updating the proxy's JS SDK dependency; it does not require a separate -operator implementation in the proxy. - -## Method - -- Baseline: PR parent `e6e7773cfbe7aa00202708d3fa49bca849797602`. +- Baseline: `e6e7773cfbe7aa00202708d3fa49bca849797602`. - Candidate: `75c1c1a1789dd53f55cdecf6dac23084ba8058f8`. -- Python 3.13.15; macOS-26.6.2-arm64-arm-64bit-Mach-O. Same interpreter and installed dependencies. -- Three independent runs per checkout, ordered base/change, change/base, - base/change. All benchmark processes ran sequentially, without concurrent tests. -- Targeted benchmark: 1,000 warmup evaluations, then seven samples of 20,000 - evaluations per client/workload per run. Reported values are medians of the - three per-run medians. Client construction, payload loading, result checks, - and cleanup are excluded. Both value and result source are checked before timing. -- Existing sync benchmark: best of five samples of 100,000 evaluations per run; - report the median of the three runs. -- Existing async benchmark: 1,000 requests, concurrency 100, simulated 1 ms sticky - storage I/O; report the median of three runs. Each request evaluates two features. -- Raw samples and candidate source hash: [JSONL](saved-group-references-v2.jsonl). - -## Existing behavior: targeted benchmark - -Times are microseconds per evaluation; positive changes mean slower. - -| Workload | Sync base | Sync change | Delta | Async base | Async change | Delta | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| default | 0.915 | 0.950 | +3.9% | 0.999 | 1.016 | +1.7% | -| experiment | 4.653 | 4.590 | -1.4% | 4.537 | 4.683 | +3.2% | -| plain-condition | 1.535 | 1.581 | +3.0% | 1.600 | 1.615 | +0.9% | -| nested-condition | 2.694 | 2.783 | +3.3% | 2.757 | 2.857 | +3.6% | -| legacy-list-10 | 1.646 | 1.701 | +3.3% | 1.764 | 1.807 | +2.4% | -| legacy-list-1000 | 6.273 | 6.199 | -1.2% | 6.226 | 6.295 | +1.1% | -| legacy-array-1000 | 8.757 | 8.485 | -3.1% | 8.687 | 8.772 | +1.0% | -| legacy-not-in-group | 1.615 | 1.664 | +3.1% | 1.763 | 1.759 | -0.2% | - -All changes were between -3.1% and +3.9%. Even the unchanged default-value -control varied by +3.9%, so differences of this size should not be interpreted -as a proven speedup or slowdown. The initial implementation showed about -+0.13 µs (+7.5%) in sync `$notInGroup`; the final comparison is +0.05 µs (+3.1%). - -## V2 payload cost - -These compare equivalent decisions on the candidate checkout: a v2 list vs. -the same legacy list, or a v2 condition group vs. its inline condition. -The old SDK cannot evaluate v2 references, so timing its no-match path would -produce a misleading comparison. - -| Equivalent workload | Client | Legacy/inline µs | V2 µs | Added µs | -| --- | --- | ---: | ---: | ---: | -| v2-list-10 | sync | 1.701 | 1.567 | -0.134 | -| v2-list-10 | async | 1.807 | 1.680 | -0.127 | -| v2-list-1000 | sync | 6.199 | 6.047 | -0.152 | -| v2-list-1000 | async | 6.295 | 6.251 | -0.044 | -| v2-condition | sync | 1.581 | 1.750 | +0.169 | -| v2-condition | async | 1.615 | 1.878 | +0.264 | -| v2-nested-condition | sync | 2.783 | 2.992 | +0.209 | -| v2-nested-condition | async | 2.857 | 3.093 | +0.236 | - -## Existing benchmark scripts - -| Sync workload | Base µs | Change µs | Delta | -| --- | ---: | ---: | ---: | -| default-value | 0.930 | 0.911 | -2.0% | -| experiment-rule | 4.146 | 4.145 | -0.0% | - -| Async workload | Base req/s | Change req/s | Throughput delta | Base p95 ms | Change p95 ms | -| --- | ---: | ---: | ---: | ---: | ---: | -| no-sticky-service | 95,458.9 | 95,855.1 | +0.4% | 0.01 | 0.01 | -| sync-sticky-service | 4,923.1 | 4,813.8 | -2.2% | 21.62 | 21.84 | -| async-sticky-service | 26,321.0 | 26,272.2 | -0.2% | 3.93 | 4.12 | -| async-hot-user | 37,538.6 | 36,426.1 | -3.0% | 2.78 | 2.84 | - -Async throughput differences were within about 3%. Event-loop lag measurements -are retained in the raw data; short runs and OS scheduling make maxima noisy. -These existing scripts do not exercise saved-group conditions, which is why -the targeted benchmark is included. +- Three sequential runs per checkout, alternating order with identical dependencies. +- Saved-group benchmarks: 1,000 warmups, seven samples of 20,000 evaluations; + results are medians of the three per-run medians. Setup and loading are excluded. +- Existing sync benchmark: best of five 100,000-evaluation samples per run. + Async benchmark: 1,000 requests, concurrency 100, simulated 1 ms sticky storage. + +| Measurement | Result | +| --- | --- | +| Existing conditions through both clients | Evaluation time changed -3.1% to +3.9% | +| Existing sync default/experiment benchmark | Evaluation time changed -2.0% / approximately 0% | +| Existing async throughput benchmark | Throughput changed -3.0% to +0.4% | +| V2 list groups vs equivalent legacy lists | No slowdown measured | +| V2 condition groups vs equivalent inline conditions | Added 0.17–0.26 µs per evaluation | + +No material regression was detected locally. The unchanged default-value control +also varied by +3.9%; these results do not establish zero regression on every +workload or Python version. ## Reproduce -Activate the development environment. Extract or check out the baseline separately, -then run the same benchmark script against each SDK path: +Use the same virtual environment's Python for both checkouts. Run sequentially, +alternate checkout order, and repeat three times: ```bash python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/baseline --legacy-only -python tests/scripts/benchmark_saved_groups.py --sdk-path "$PWD" --legacy-only -python tests/scripts/benchmark_saved_groups.py # includes v2 workloads -python tests/scripts/benchmark_eval_overhead.py 100000 -PYTHONPATH=. python tests/scripts/benchmark_async_client.py +python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/candidate --legacy-only +python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/candidate ``` -For the existing scripts, run from each checkout with the same absolute path to -the virtual environment's Python, and set `PYTHONPATH=.` so editable-install -resolution does not accidentally benchmark the wrong checkout. Alternate the -checkout order and repeat; do not run the two benchmarks concurrently. +Run the existing benchmarks from each checkout with `PYTHONPATH=.` and the same +absolute path to Python: + +```bash +PYTHONPATH=. python tests/scripts/benchmark_eval_overhead.py 100000 +PYTHONPATH=. python tests/scripts/benchmark_async_client.py +``` diff --git a/tests/scripts/README.md b/tests/scripts/README.md index c244c73..25e32b4 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -9,20 +9,9 @@ Also in this directory (documented in their own docstrings): `GrowthBookClient` with sync/async sticky bucket services. Reports throughput, latency percentiles, and event-loop lag. Run with `PYTHONPATH=. python3 tests/scripts/benchmark_async_client.py`. -- `benchmark_saved_groups.py` — sync and async evaluation benchmarks for - ordinary conditions, legacy groups, and v2 list/condition groups. Checks - results before timing and emits JSON with all samples. Compare the same - workload on two checkouts with the same Python environment: - - ```bash - python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/base --legacy-only - python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/change --legacy-only - python tests/scripts/benchmark_saved_groups.py # includes v2 workloads - ``` - - Run sequentially, alternate checkout order, and repeat before drawing - conclusions. `--iterations` and `--rounds` control sample size. These are - local CPU measurements, not network or production throughput guarantees. +- `benchmark_saved_groups.py` — sync/async condition and saved-group benchmarks. + Run `python3 tests/scripts/benchmark_saved_groups.py`; use `--help` for options. + See the [comparison results and commands](../../docs/benchmarks/saved-group-references-v2.md). - `check_corpus_freshness.py` — cases.json drift check vs the JS SDK (runs as its own CI job). From f6e6101959df2ab58bef30af19c378002707271a Mon Sep 17 00:00:00 2001 From: Andy White Date: Mon, 28 Sep 2026 16:32:06 -0600 Subject: [PATCH 5/6] fix: fail closed on condition overflow and compare container membership strictly --- growthbook/core.py | 29 +++++++++++++++------ tests/test_saved_groups.py | 53 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/growthbook/core.py b/growthbook/core.py index 26ead4e..b461886 100644 --- a/growthbook/core.py +++ b/growthbook/core.py @@ -33,6 +33,18 @@ def evalCondition( attributes: Dict[str, Any], condition: Dict[str, Any], savedGroups: Optional[Dict[str, Any]] = None, visited: Optional[Set[str]] = None, ) -> bool: + try: + return _eval_condition(attributes, condition, savedGroups, visited) + except RecursionError: + # Fail the entire condition so $not cannot turn a stack overflow into a match. + return False + + +def _eval_condition( + attributes: Dict[str, Any], condition: Dict[str, Any], + savedGroups: Optional[Dict[str, Any]], visited: Optional[Set[str]], +) -> bool: + """Evaluate recursively, leaving stack overflow handling to the outer caller.""" for key, value in condition.items(): if key == "$or": if not evalOr(attributes, value, savedGroups, visited): @@ -44,7 +56,7 @@ def evalCondition( if not evalAnd(attributes, value, savedGroups, visited): return False elif key == "$not": - if evalCondition(attributes, value, savedGroups, visited): + if _eval_condition(attributes, value, savedGroups, visited): return False elif key == "$savedGroup": if not _eval_saved_group(attributes, value, savedGroups, visited): @@ -86,9 +98,9 @@ def _eval_saved_group( return False # Allocate the branch guard only when entering a condition group. next_visited = visited | {group_id} if visited else {group_id} - return evalCondition(attributes, condition, saved_groups, next_visited) - except (TypeError, ValueError, AttributeError, IndexError, RecursionError): - # Malformed payloads and Python's own stack limit must not escape evaluation. + return _eval_condition(attributes, condition, saved_groups, next_visited) + except (TypeError, ValueError, AttributeError, IndexError): + # Malformed payloads must not escape evaluation. return False return False @@ -115,7 +127,7 @@ def evalOr( return True for condition in conditions: - if evalCondition(attributes, condition, savedGroups, visited): + if _eval_condition(attributes, condition, savedGroups, visited): return True return False @@ -125,7 +137,7 @@ def evalAnd( visited: Optional[Set[str]] = None, ) -> bool: for condition in conditions: - if not evalCondition(attributes, condition, savedGroups, visited): + if not _eval_condition(attributes, condition, savedGroups, visited): return False return True @@ -190,7 +202,7 @@ def elemMatch( if evalConditionValue(condition, item, savedGroups, visited=visited): return True else: - if evalCondition(item, condition, savedGroups, visited): + if _eval_condition(item, condition, savedGroups, visited): return True return False @@ -423,7 +435,8 @@ def case_fold(val: Any) -> Any: return bool(set(conditionValue) & set(attributeValue)) except TypeError: # JSON arrays may contain objects or arrays, which cannot be hashed. - return any(value in conditionValue for value in attributeValue) + return any(_js_strict_equal(value, expected) + for value in attributeValue for expected in conditionValue) return attributeValue in conditionValue def isInAll( diff --git a/tests/test_saved_groups.py b/tests/test_saved_groups.py index 191d5e4..8ffbdf3 100644 --- a/tests/test_saved_groups.py +++ b/tests/test_saved_groups.py @@ -191,6 +191,43 @@ def test_cycle_longer_than_python_stack_fails_closed(): assert not evalCondition({}, {"$savedGroup": {"id": "0"}}, groups) +@pytest.mark.asyncio +@pytest.mark.parametrize("condition", [ + {"$savedGroup": {"id": "0"}}, + {"$not": {"$savedGroup": {"id": "0"}}}, + {"$and": [{"$savedGroup": {"id": "0"}}]}, + {"$or": [{"$savedGroup": {"id": "0"}}, {}]}, + {"$nor": [{"$savedGroup": {"id": "0"}}]}, + {"items": {"$elemMatch": {"marker": True, "$savedGroup": {"id": "0"}}}}, + {"items": {"$not": {"$elemMatch": {"marker": True, "$savedGroup": {"id": "0"}}}}}, +], ids=["reference", "not", "and", "or", "nor", "elemMatch", "value-not"]) +async def test_recursion_overflow_rejects_the_entire_rule(condition): + count = sys.getrecursionlimit() + 50 + groups = {str(i): {"type": "condition", "condition": {"$savedGroup": {"id": str(i + 1)}}} + for i in range(count)} + groups[str(count)] = {"type": "condition", "condition": {}} + attributes = {"id": "u1", "items": [{"marker": True}]} + payload = {"savedGroups": groups, "features": {"flag": { + "defaultValue": True, "rules": [{"condition": condition, "force": True}, {"force": False}], + }}} + experiment = Experiment(key="deep", variations=[0, 1], condition=condition) + sync_client = GrowthBook(attributes=dict(attributes)) + async_client = GrowthBookClient(Options()) + try: + sync_client.set_payload(payload) + await async_client.set_payload(payload) + user = UserContext(attributes=attributes) + assert evalCondition(attributes, condition, groups) is False + assert sync_client.is_on("flag") is False + assert await async_client.is_on("flag", user) is False + assert sync_client.run(experiment).inExperiment is False + assert (await async_client.run(experiment, user)).inExperiment is False + assert evalCondition(attributes, {"$or": [{}, condition]}, groups) is True + finally: + sync_client.destroy() + await async_client.close() + + @pytest.mark.parametrize("marker", ["__sgInvalid__", "__sgUnknown__", "__sgCycle__", "__sgMaxDepth__"]) def test_error_markers_keep_boolean_semantics(marker): condition = {marker: "g"} @@ -215,6 +252,22 @@ def test_array_membership_handles_unhashable_attributes(): assert evalCondition(attributes, {"$savedGroup": {"id": "g"}}, groups) +@pytest.mark.parametrize("value", [{"id": "u1"}, ["u1"]]) +@pytest.mark.parametrize("primitive_match", [False, True]) +def test_container_membership_does_not_use_structural_equality(value, primitive_match): + attributes = {"id": [deepcopy(value), "u1"]} + values = [deepcopy(value), "u1" if primitive_match else "u2"] + for entry in [values, {"type": "list", "attributeKey": "id", "values": values}]: + groups = {"g": entry} + for operator, expected in [("$in", primitive_match), ("$nin", not primitive_match)]: + assert evalCondition(attributes, {"id": {operator: values}}) is expected + assert evalCondition(attributes, {"id": {"$inGroup": "g"}}, groups) is primitive_match + assert evalCondition(attributes, {"id": {"$notInGroup": "g"}}, groups) is (not primitive_match) + if isinstance(entry, dict): + assert evalCondition(attributes, {"$savedGroup": {"id": "g"}}, groups) is primitive_match + + + @pytest.mark.asyncio @pytest.mark.parametrize("attributes, expected", [ ({"id": "u1", "plan": "pro"}, True), From e964b5d7b3869cffef76fd68bfbb170ad797cab7 Mon Sep 17 00:00:00 2001 From: Andy White Date: Wed, 30 Sep 2026 16:46:38 -0600 Subject: [PATCH 6/6] fix: reject condition errors and distinguish boolean membership --- docs/benchmarks/saved-group-references-v2.md | 42 ------------ growthbook/core.py | 72 +++++++++++++------- tests/scripts/README.md | 9 ++- tests/test_saved_groups.py | 45 ++++++++++++ 4 files changed, 99 insertions(+), 69 deletions(-) delete mode 100644 docs/benchmarks/saved-group-references-v2.md diff --git a/docs/benchmarks/saved-group-references-v2.md b/docs/benchmarks/saved-group-references-v2.md deleted file mode 100644 index 0030dcd..0000000 --- a/docs/benchmarks/saved-group-references-v2.md +++ /dev/null @@ -1,42 +0,0 @@ -# Saved group references v2 benchmarks - -Measured 2026-09-25 on Python 3.13.15, macOS 26.6.2 ARM64: - -- Baseline: `e6e7773cfbe7aa00202708d3fa49bca849797602`. -- Candidate: `75c1c1a1789dd53f55cdecf6dac23084ba8058f8`. -- Three sequential runs per checkout, alternating order with identical dependencies. -- Saved-group benchmarks: 1,000 warmups, seven samples of 20,000 evaluations; - results are medians of the three per-run medians. Setup and loading are excluded. -- Existing sync benchmark: best of five 100,000-evaluation samples per run. - Async benchmark: 1,000 requests, concurrency 100, simulated 1 ms sticky storage. - -| Measurement | Result | -| --- | --- | -| Existing conditions through both clients | Evaluation time changed -3.1% to +3.9% | -| Existing sync default/experiment benchmark | Evaluation time changed -2.0% / approximately 0% | -| Existing async throughput benchmark | Throughput changed -3.0% to +0.4% | -| V2 list groups vs equivalent legacy lists | No slowdown measured | -| V2 condition groups vs equivalent inline conditions | Added 0.17–0.26 µs per evaluation | - -No material regression was detected locally. The unchanged default-value control -also varied by +3.9%; these results do not establish zero regression on every -workload or Python version. - -## Reproduce - -Use the same virtual environment's Python for both checkouts. Run sequentially, -alternate checkout order, and repeat three times: - -```bash -python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/baseline --legacy-only -python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/candidate --legacy-only -python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/candidate -``` - -Run the existing benchmarks from each checkout with `PYTHONPATH=.` and the same -absolute path to Python: - -```bash -PYTHONPATH=. python tests/scripts/benchmark_eval_overhead.py 100000 -PYTHONPATH=. python tests/scripts/benchmark_async_client.py -``` diff --git a/growthbook/core.py b/growthbook/core.py index b461886..e5f7310 100644 --- a/growthbook/core.py +++ b/growthbook/core.py @@ -35,8 +35,8 @@ def evalCondition( ) -> bool: try: return _eval_condition(attributes, condition, savedGroups, visited) - except RecursionError: - # Fail the entire condition so $not cannot turn a stack overflow into a match. + except (TypeError, ValueError, AttributeError, IndexError, RecursionError): + # Fail the entire condition so $not cannot turn an evaluation error into a match. return False @@ -44,7 +44,7 @@ def _eval_condition( attributes: Dict[str, Any], condition: Dict[str, Any], savedGroups: Optional[Dict[str, Any]], visited: Optional[Set[str]], ) -> bool: - """Evaluate recursively, leaving stack overflow handling to the outer caller.""" + """Evaluate recursively, leaving evaluation error handling to the outer caller.""" for key, value in condition.items(): if key == "$or": if not evalOr(attributes, value, savedGroups, visited): @@ -85,23 +85,19 @@ def _eval_saved_group( if not isinstance(entry, dict): return False - try: - if entry.get("type") == "list": - key = reference.get("attributeKey", entry.get("attributeKey")) - values = entry.get("values") - if not isinstance(key, str) or not isinstance(values, list): - return False - return isIn(values, getPath(attributes, key)) - if entry.get("type") == "condition": - condition = entry.get("condition") - if not isinstance(condition, dict): - return False - # Allocate the branch guard only when entering a condition group. - next_visited = visited | {group_id} if visited else {group_id} - return _eval_condition(attributes, condition, saved_groups, next_visited) - except (TypeError, ValueError, AttributeError, IndexError): - # Malformed payloads must not escape evaluation. - return False + if entry.get("type") == "list": + key = reference.get("attributeKey", entry.get("attributeKey")) + values = entry.get("values") + if not isinstance(key, str) or not isinstance(values, list): + return False + return isIn(values, getPath(attributes, key)) + if entry.get("type") == "condition": + condition = entry.get("condition") + if not isinstance(condition, dict): + return False + # Allocate the branch guard only when entering a condition group. + next_visited = visited | {group_id} if visited else {group_id} + return _eval_condition(attributes, condition, saved_groups, next_visited) return False @@ -423,22 +419,46 @@ def case_fold(val: Any) -> Any: # Do an intersection if attribute is an array (insensitive) if isinstance(attributeValue, list): return any( - case_fold(el) == case_fold(exp) + case_fold(el) == case_fold(exp) and isinstance(el, bool) == isinstance(exp, bool) for el in attributeValue for exp in conditionValue ) - return any(case_fold(attributeValue) == case_fold(exp) for exp in conditionValue) + return any( + case_fold(attributeValue) == case_fold(exp) + and isinstance(attributeValue, bool) == isinstance(exp, bool) + for exp in conditionValue + ) - # Case-sensitive behavior (original) if isinstance(attributeValue, list): try: - return bool(set(conditionValue) & set(attributeValue)) + matches = set(conditionValue) & set(attributeValue) + # Python collapses booleans with 0/1; any other match is unambiguous. + if not matches or matches.difference((0, 1)): + return bool(matches) + return any(_contains_boolean_or_number(conditionValue, value) + for value in attributeValue if value == 0 or value == 1) except TypeError: # JSON arrays may contain objects or arrays, which cannot be hashed. - return any(_js_strict_equal(value, expected) - for value in attributeValue for expected in conditionValue) + pass + return any(_js_strict_equal(value, expected) + for value in attributeValue for expected in conditionValue) + if attributeValue == 0 or attributeValue == 1: + return _contains_boolean_or_number(conditionValue, attributeValue) return attributeValue in conditionValue + +def _contains_boolean_or_number(values: List[Any], value: Any) -> bool: + """Find 0/1 or False/True without Python's boolean/number equality aliasing.""" + start = 0 + while True: + try: + index = values.index(value, start) + except ValueError: + return False + if isinstance(values[index], bool) == isinstance(value, bool): + return True + start = index + 1 + def isInAll( conditionValue: List[Any], attributeValue: Any, savedGroups: Optional[Dict[str, Any]], insensitive: bool = False, visited: Optional[Set[str]] = None, diff --git a/tests/scripts/README.md b/tests/scripts/README.md index 25e32b4..eb7c1c5 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -11,7 +11,14 @@ Also in this directory (documented in their own docstrings): Run with `PYTHONPATH=. python3 tests/scripts/benchmark_async_client.py`. - `benchmark_saved_groups.py` — sync/async condition and saved-group benchmarks. Run `python3 tests/scripts/benchmark_saved_groups.py`; use `--help` for options. - See the [comparison results and commands](../../docs/benchmarks/saved-group-references-v2.md). + Compare checkouts using the same Python environment: + + ```bash + python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/base --legacy-only + python tests/scripts/benchmark_saved_groups.py --sdk-path /path/to/change --legacy-only + ``` + + Run sequentially, alternate checkout order, and repeat. Keep results in the PR description. - `check_corpus_freshness.py` — cases.json drift check vs the JS SDK (runs as its own CI job). diff --git a/tests/test_saved_groups.py b/tests/test_saved_groups.py index 8ffbdf3..f7d0aa1 100644 --- a/tests/test_saved_groups.py +++ b/tests/test_saved_groups.py @@ -115,6 +115,51 @@ def test_malformed_entries_fail_closed(entry): assert evalCondition(attributes, {"id": {"$notInGroup": "missing"}}, groups) is True +@pytest.mark.asyncio +@pytest.mark.parametrize("invalid", [{"$or": 5}, {"$not": None}, {"id": {"": True}}]) +async def test_condition_errors_fail_closed_under_negation(invalid): + groups = {"bad": {"type": "condition", "condition": invalid}} + sync_client = GrowthBook(attributes={"id": "u1"}) + async_client = GrowthBookClient(Options()) + try: + for target in (invalid, {"$savedGroup": {"id": "bad"}}): + condition = {"$not": target} + payload = {"savedGroups": groups, "features": {"flag": { + "defaultValue": False, "rules": [{"condition": condition, "force": True}], + }}} + sync_client.set_payload(payload) + await async_client.set_payload(payload) + assert not evalCondition({"id": "u1"}, condition, groups) + assert not sync_client.is_on("flag") + assert not await async_client.is_on("flag", UserContext(attributes={"id": "u1"})) + finally: + sync_client.destroy() + await async_client.close() + + +@pytest.mark.parametrize("as_array", [False, True]) +@pytest.mark.parametrize("attribute, values, expected", [ + (True, [1], False), (1, [True], False), + (False, [0], False), (0, [False], False), + (True, [1.0], False), (1.0, [True], False), + (True, [True], True), (False, [False], True), + (1, [1.0], True), (0, [0.0], True), + (True, [1, True], True), (1, [True, 1], True), + (False, [0, False], True), (0, [False, 0], True), +]) +def test_membership_distinguishes_booleans_from_numbers(as_array, attribute, values, expected): + attributes = {"id": [attribute] if as_array else attribute} + for operator in ("$in", "$ini", "$nin", "$nini"): + result = not expected if operator in ("$nin", "$nini") else expected + assert evalCondition(attributes, {"id": {operator: values}}) is result + for entry in (values, {"type": "list", "attributeKey": "id", "values": values}): + groups = {"g": entry} + assert evalCondition(attributes, {"id": {"$inGroup": "g"}}, groups) is expected + assert evalCondition(attributes, {"id": {"$notInGroup": "g"}}, groups) is (not expected) + if isinstance(entry, dict): + assert evalCondition(attributes, {"$savedGroup": {"id": "g"}}, groups) is expected + + @pytest.mark.parametrize("groups", [None, {}]) def test_legacy_operators_without_saved_groups(groups): assert evalCondition({"id": "u1"}, {"id": {"$inGroup": "missing"}}, groups) is False