Skip to content

Commit 0124739

Browse files
committed
test: Run the override vectors and the event marking scenarios on the async client
1 parent 1121828 commit 0124739

3 files changed

Lines changed: 303 additions & 1 deletion

File tree

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
"""
2+
Runs the OVERRIDE specification test vectors through the async client. The vectors and the
3+
checks are those of the sync runner: each vector sets up LaunchDarkly data, an override layer,
4+
and an initialization state, and the test evaluates one flag through the full client stack.
5+
"""
6+
from typing import Any, Dict
7+
8+
import pytest
9+
10+
from ldclient.async_client import AsyncLDClient
11+
from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig
12+
from ldclient.context import Context
13+
from ldclient.impl.events.types import EventInputEvaluation
14+
from ldclient.testing.mock_async_components import MockAsyncEventProcessor
15+
from ldclient.testing.mock_components import (
16+
MockDataSourceBuilder,
17+
MockOverrideSource
18+
)
19+
from ldclient.testing.test_async_client_overrides import (
20+
AsyncHangingSynchronizer,
21+
AsyncStaticInitializer
22+
)
23+
from ldclient.testing.test_ldclient_override_vectors import (
24+
assert_reason,
25+
load_vectors,
26+
override_flags,
27+
vector_id
28+
)
29+
30+
31+
async def make_client(vector: Dict[str, Any]) -> AsyncLDClient:
32+
source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {})))
33+
ld_data = vector['launchDarklyData']
34+
if ld_data['initialized']:
35+
initializer = AsyncStaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {}))
36+
datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder)
37+
start_wait = 5
38+
else:
39+
# With no sources at all, the client would consider cached data available rather than
40+
# applying its not-initialized handling. A synchronizer that never delivers avoids that.
41+
datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder)
42+
start_wait = 0
43+
config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor())
44+
client = AsyncLDClient(config)
45+
await client.start(start_wait=start_wait)
46+
return client
47+
48+
49+
@pytest.mark.asyncio
50+
@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id)
51+
async def test_override_spec_vector(vector: Dict[str, Any]):
52+
client = await make_client(vector)
53+
try:
54+
assert await client.is_initialized() is vector['launchDarklyData']['initialized']
55+
evaluate = vector['evaluate']
56+
detail = await client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue'])
57+
58+
expect = vector['expect']
59+
assert detail.value == expect['value'], "value"
60+
assert detail.variation_index == expect['variationIndex'], "variationIndex"
61+
assert_reason(expect['reason'], detail.reason)
62+
63+
# summaryOverrideAffected is the marking the client hands to the event processor for this
64+
# evaluation. The event processor keys individual-event suppression and the summary
65+
# counter marker on that scalar, not on the reason.
66+
if 'summaryOverrideAffected' in expect:
67+
records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']]
68+
assert len(records) == 1, "expected exactly one evaluation record for the flag"
69+
assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected"
70+
finally:
71+
await client.close()

‎ldclient/testing/test_async_client_overrides.py‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
client tests.
44
"""
55
import asyncio
6+
import json
67
import threading
78
import time
89
from typing import Any, Dict, List, Optional
@@ -14,11 +15,17 @@
1415
from ldclient.async_feature_store import AsyncInMemoryFeatureStore
1516
from ldclient.context import Context
1617
from ldclient.evaluation import EvaluationDetail
18+
from ldclient.hook import AsyncHook, EvaluationSeriesContext, Metadata
1719
from ldclient.impl.aio.concurrency import AsyncEvent
20+
from ldclient.impl.events.async_event_processor import (
21+
DefaultAsyncEventProcessor
22+
)
1823
from ldclient.impl.events.types import EventInputEvaluation
1924
from ldclient.impl.integrations.files.filedata import make_flag_with_value
2025
from ldclient.interfaces import AsyncFeatureStore, DataStoreMode
26+
from ldclient.migrations import Stage
2127
from ldclient.testing.builders import FlagBuilder
28+
from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp
2229
from ldclient.testing.mock_async_components import MockAsyncEventProcessor
2330
from ldclient.testing.mock_components import (
2431
FailingOverrideSourceBuilder,
@@ -345,3 +352,189 @@ async def test_all_flags_state_turns_off_event_tracking_for_override_affected_fl
345352
assert state.get_flag_value('overridden-flag') is True
346353
finally:
347354
await client.close()
355+
356+
357+
@pytest.mark.asyncio
358+
async def test_override_source_is_not_started_when_offline():
359+
source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)})
360+
config = AsyncConfig('SDK_KEY', datasystem_config=AsyncDataSystemConfig(override_source=source.builder), offline=True)
361+
client = AsyncLDClient(config)
362+
await client.start(start_wait=0)
363+
try:
364+
assert source.start_count == 0
365+
assert await client.variation('overridden-flag', user, False) is False
366+
finally:
367+
await client.close()
368+
369+
370+
def test_override_update_after_the_client_and_its_loop_are_gone_is_dropped():
371+
source = MockOverrideSource()
372+
373+
async def run_client() -> None:
374+
client = await make_uninitialized_client(source)
375+
client.flag_tracker.add_listener(lambda change: None)
376+
await client.close()
377+
378+
asyncio.run(run_client())
379+
# A reload that finishes after the client closed finds the event loop closed. The
380+
# notification has nobody left to reach and is dropped.
381+
source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {})
382+
383+
384+
# ---------------------------------------------------------------------------
385+
# Events
386+
# ---------------------------------------------------------------------------
387+
388+
def tracked_bool_flag(key: str) -> FlagBuilder:
389+
return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True)
390+
391+
392+
def evaluation_events_by_key(client: AsyncLDClient) -> Dict[str, EventInputEvaluation]:
393+
"""The evaluation records the client handed to the event processor, keyed by flag key."""
394+
processor: Any = client._event_processor
395+
return {event.key: event for event in processor.events if isinstance(event, EventInputEvaluation)}
396+
397+
398+
async def raise_evaluation_failure(*args):
399+
raise RuntimeError("evaluation failure")
400+
401+
402+
@pytest.mark.asyncio
403+
async def test_overridden_prerequisite_marks_the_dependent_evaluation_records():
404+
# top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden)
405+
# --> plain-flag (LaunchDarkly)
406+
# The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override.
407+
ld_data = {
408+
'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(),
409+
'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(),
410+
'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(),
411+
'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(),
412+
}
413+
source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()})
414+
client = await make_initialized_client(ld_data, source)
415+
try:
416+
detail = await client.variation_detail('top-flag', user, False)
417+
assert detail.value is True
418+
assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True}
419+
420+
records = evaluation_events_by_key(client)
421+
assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag']
422+
assert records['top-flag'].override_affected is True
423+
assert records['mid-flag'].override_affected is True
424+
assert records['leaf-flag'].override_affected is True
425+
assert records['plain-flag'].override_affected is False
426+
assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True}
427+
assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'}
428+
finally:
429+
await client.close()
430+
431+
432+
@pytest.mark.asyncio
433+
async def test_wrong_type_result_of_overridden_flag_stays_marked():
434+
details = []
435+
436+
class CapturingHook(AsyncHook):
437+
@property
438+
def metadata(self) -> Metadata:
439+
return Metadata(name='capturing-hook')
440+
441+
async def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict:
442+
return data
443+
444+
async def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict:
445+
details.append(detail)
446+
return data
447+
448+
source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')})
449+
datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder)
450+
config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor(), hooks=[CapturingHook()])
451+
client = AsyncLDClient(config)
452+
await client.start(start_wait=0)
453+
try:
454+
stage, _ = await client.migration_variation('overridden-flag', user, Stage.OFF)
455+
assert stage == Stage.OFF
456+
assert len(details) == 1
457+
assert details[0].value == 'off'
458+
assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True}
459+
finally:
460+
await client.close()
461+
462+
463+
@pytest.mark.asyncio
464+
async def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch):
465+
source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()})
466+
client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source)
467+
try:
468+
monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure)
469+
470+
# The failure of the override flag is marked. The failure of the ordinary flag is not.
471+
detail = await client.variation_detail('overridden-flag', user, 'default')
472+
assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True})
473+
detail = await client.variation_detail('plain-flag', user, 'default')
474+
assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'})
475+
476+
# The marked record produces no individual event. The ordinary record keeps its tracking.
477+
records = evaluation_events_by_key(client)
478+
assert records['overridden-flag'].override_affected is True
479+
assert records['plain-flag'].override_affected is False
480+
assert records['plain-flag'].track_events is True
481+
finally:
482+
await client.close()
483+
484+
485+
@pytest.mark.asyncio
486+
async def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch):
487+
source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()})
488+
client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source)
489+
try:
490+
monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure)
491+
state = await client.all_flags_state(user, with_reasons=True)
492+
493+
# The failed override flag stays in the state with a marked reason and no tracking
494+
# fields. The failed ordinary flag keeps its tracking fields.
495+
assert state.valid is True
496+
flags_state = state.to_json_dict()['$flagsState']
497+
assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}
498+
assert 'trackEvents' not in flags_state['overridden-flag']
499+
assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}
500+
assert flags_state['plain-flag']['trackEvents'] is True
501+
finally:
502+
await client.close()
503+
504+
505+
@pytest.mark.asyncio
506+
async def test_override_affected_evaluations_appear_only_in_summary_output():
507+
# End to end through the real event processor: the overridden flag requests individual
508+
# feature events and debug events, and an ordinary flag requests feature events.
509+
debug_until = int(time.time() * 1000) + 100000
510+
overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict()
511+
normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict()
512+
source = MockOverrideSource(flags={'flag-tracked-override': overridden})
513+
mock_http = MockAioHttp()
514+
515+
initializer = AsyncStaticInitializer({'flag-normal': normal}, {})
516+
datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder)
517+
config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultAsyncEventProcessor(config, mock_http))
518+
client = AsyncLDClient(config)
519+
await client.start(start_wait=5)
520+
try:
521+
assert await client.is_initialized() is True
522+
for _ in range(2):
523+
assert await client.variation('flag-tracked-override', user, 'default1') == 'override-value'
524+
assert await client.variation('flag-normal', user, 'default2') == 'normal-value'
525+
assert await client._event_processor.flush_and_wait(5) is True
526+
finally:
527+
await client.close()
528+
529+
assert mock_http.request_data is not None
530+
output = json.loads(mock_http.request_data)
531+
kinds = sorted(e['kind'] for e in output)
532+
assert kinds == ['feature', 'index', 'summary']
533+
feature = [e for e in output if e['kind'] == 'feature'][0]
534+
assert feature['key'] == 'flag-normal'
535+
summary = [e for e in output if e['kind'] == 'summary'][0]
536+
assert summary['features']['flag-tracked-override']['default'] == 'default1'
537+
assert summary['features']['flag-tracked-override']['counters'] == [
538+
{'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True}
539+
]
540+
assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}]

‎ldclient/testing/test_ldclient_overrides.py‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@ def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags():
464464
assert flags_state['overridden-flag']['version'] == 7
465465

466466

467-
def test_all_flags_state_keeps_details_of_override_affected_flags_when_details_only_for_tracked_flags():
467+
def test_all_flags_state_omits_details_of_override_affected_flags_when_details_only_for_tracked_flags():
468468
# With details only for tracked flags, an override-affected flag counts as untracked, so its
469469
# version and reason are omitted like any other untracked flag, and its value stays.
470470
overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict()
@@ -503,6 +503,44 @@ def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict,
503503
assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True}
504504

505505

506+
def raise_evaluation_failure(*args):
507+
raise RuntimeError("evaluation failure")
508+
509+
510+
def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch):
511+
source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()})
512+
with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client:
513+
monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure)
514+
515+
# The failure of the override flag is marked. The failure of the ordinary flag is not.
516+
detail = client.variation_detail('overridden-flag', user, 'default')
517+
assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True})
518+
detail = client.variation_detail('plain-flag', user, 'default')
519+
assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'})
520+
521+
# The marked record produces no individual event. The ordinary record keeps its tracking.
522+
records = evaluation_events_by_key(client)
523+
assert records['overridden-flag'].override_affected is True
524+
assert records['plain-flag'].override_affected is False
525+
assert records['plain-flag'].track_events is True
526+
527+
528+
def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch):
529+
source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()})
530+
with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client:
531+
monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure)
532+
state = client.all_flags_state(user, with_reasons=True)
533+
534+
# The failed override flag stays in the state with a marked reason and no tracking
535+
# fields. The failed ordinary flag keeps its tracking fields.
536+
assert state.valid is True
537+
flags_state = state.to_json_dict()['$flagsState']
538+
assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}
539+
assert 'trackEvents' not in flags_state['overridden-flag']
540+
assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}
541+
assert flags_state['plain-flag']['trackEvents'] is True
542+
543+
506544
def test_override_affected_evaluations_appear_only_in_summary_output():
507545
# End to end through the real event processor: the overridden flag requests individual
508546
# feature events and debug events, and an ordinary flag requests feature events.

0 commit comments

Comments
 (0)