From 6f4414b100e017b4facae1c0ab5b0bcd3003be0d Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Fri, 16 Oct 2026 18:41:21 +0300 Subject: [PATCH 1/2] perf: call once before building the retryer (#71) A first attempt that succeeds no longer enters tenacity's attempt loop, so postgres_retry costs about 0.2 us per call instead of 22 us. After a retriable first failure, the retryer replays that error as attempt 1, so waits, attempt counts and logging stay the same. --- db_retry/retry.py | 21 +++++++++++++++++++-- tests/test_retry.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/db_retry/retry.py b/db_retry/retry.py index 2ac62cf..6800b52 100644 --- a/db_retry/retry.py +++ b/db_retry/retry.py @@ -42,14 +42,31 @@ def postgres_retry( def decorator(f: _Func[P, T]) -> _Func[P, T]: @functools.wraps(f) async def wrapped_method(*args: P.args, **kwargs: P.kwargs) -> T: + try: + return await f(*args, **kwargs) + except Exception as exception: + attempts = retries if retries is not None else settings.get_retries_number() + if attempts <= 1 or not is_retriable(exception): + _log_and_decide(exception) + raise + first_error = exception + + @functools.wraps(f) + async def replay_first_error_then_call(*args: P.args, **kwargs: P.kwargs) -> T: + nonlocal first_error + if first_error is not None: + error, first_error = first_error, None + raise error + return await f(*args, **kwargs) + retryer = tenacity.AsyncRetrying( - stop=tenacity.stop_after_attempt(retries if retries is not None else settings.get_retries_number()), + stop=tenacity.stop_after_attempt(attempts), wait=tenacity.wait_exponential_jitter(), retry=tenacity.retry_if_exception(_log_and_decide), reraise=True, before=tenacity.before_log(logger, logging.DEBUG), ) - return await retryer(f, *args, **kwargs) + return await retryer(replay_first_error_then_call, *args, **kwargs) return wrapped_method diff --git a/tests/test_retry.py b/tests/test_retry.py index d9b2b90..bb3ad18 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -1,9 +1,11 @@ import asyncio import typing +from unittest import mock import asyncpg import pytest import sqlalchemy +import tenacity from sqlalchemy.exc import DBAPIError from sqlalchemy.ext import asyncio as sa_async @@ -208,3 +210,39 @@ async def select_one() -> int | None: assert await select_one() == 1 assert attempts == 2 # noqa: PLR2004 # the closed connection, then a fresh one + + +async def test_a_first_attempt_that_succeeds_never_enters_tenacity(monkeypatch: pytest.MonkeyPatch) -> None: + """INVARIANT: the success path is one direct call; tenacity is built only after a retriable error. + + Broken by going back to building ``tenacity.AsyncRetrying`` before the first attempt. Every + behavioural test still passes then, because the retryer is correct, only slow: its attempt loop + costs about 20 µs per call, more than ``pool_pre_ping`` costs against a local database, and + almost every call succeeds first time (#71). + """ + retryer_class: typing.Final = mock.Mock(wraps=tenacity.AsyncRetrying) + monkeypatch.setattr(tenacity, "AsyncRetrying", retryer_class) + + @postgres_retry + async def _succeeds() -> str: + return "ok" + + assert await _succeeds() == "ok" + retryer_class.assert_not_called() + + +async def test_a_retriable_first_failure_waits_once_then_succeeds(monkeypatch: pytest.MonkeyPatch) -> None: + waits: typing.Final = _record_backoff(monkeypatch) + attempts = 0 + + @postgres_retry(retries=3) + async def _fails_once() -> int: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise _retriable_error() + return attempts + + assert await _fails_once() == 2 # noqa: PLR2004 # the failed attempt, then the one that succeeds + assert len(waits) == 1 + assert waits[0] > 0 From 00a48ffa38fb0e2eea0dbe7b76af047896e03bb1 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Fri, 16 Oct 2026 18:50:42 +0300 Subject: [PATCH 2/2] perf: replace tenacity with a plain retry loop The loop keeps the attempt count, the predicate-first decision, the jittered exponential wait and re-raising, and drops the tenacity dependency. --- AGENTS.md | 4 ++-- CONTEXT.md | 2 +- README.md | 1 - db_retry/retry.py | 43 ++++++++++++++++--------------------------- pyproject.toml | 1 - tests/test_retry.py | 34 ++++++---------------------------- 6 files changed, 25 insertions(+), 60 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0b0846a..dc28ad6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,8 +8,8 @@ repository. `db-retry` is a library of retry, connection, and transaction helpers for PostgreSQL applications built on SQLAlchemy's asyncio extension and asyncpg; [`CONTEXT.md`](CONTEXT.md) opens with what it does and owns the vocabulary — read it before naming a concept in code, a test name, or an issue -title. The words most easily got wrong here are the ones this package shares with PostgreSQL and -with tenacity: what "retriable" covers, what a "retry" counts, and what "primary" does not mean. +title. The words most easily got wrong here are the ones this package shares with PostgreSQL: what +"retriable" covers, what a "retry" counts, and what "primary" does not mean. ## Commands diff --git a/CONTEXT.md b/CONTEXT.md index cb6f03b..edfa462 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -22,7 +22,7 @@ is safe, and the two come apart: `StatementCompletionUnknownError` (`40003`) is **Attempt**: One call of the wrapped coroutine. `postgres_retry(retries=N)` bounds *attempts*, not retries — `N` -goes to tenacity's `stop_after_attempt`, so `retries=1` runs the body once and never re-runs it, +caps the calls of the body, so `retries=1` runs the body once and never re-runs it, and the `DB_RETRY_RETRIES_NUMBER` default of `3` allows two re-runs. _Avoid_: retry, as a count. The knob is spelled `retries` for callers and cannot be renamed without breaking them, but every number in this package counts attempts. diff --git a/README.md b/README.md index fe9337b..8ede908 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,6 @@ export DB_RETRY_RETRIES_NUMBER=5 - Python 3.11+ - SQLAlchemy with asyncio support - asyncpg PostgreSQL driver -- tenacity for retry logic ## 📦 [PyPI](https://pypi.org/project/db-retry) diff --git a/db_retry/retry.py b/db_retry/retry.py index 6800b52..fdd8855 100644 --- a/db_retry/retry.py +++ b/db_retry/retry.py @@ -1,9 +1,9 @@ +import asyncio import functools import logging +import random import typing -import tenacity - from db_retry import settings from db_retry.retriable import is_retriable @@ -19,6 +19,10 @@ def _log_and_decide(exception: BaseException) -> bool: return False +def _backoff(attempt: int) -> float: + return 2 ** (attempt - 1) + random.uniform(0, 1) # noqa: S311 + + P = typing.ParamSpec("P") T = typing.TypeVar("T") @@ -42,31 +46,16 @@ def postgres_retry( def decorator(f: _Func[P, T]) -> _Func[P, T]: @functools.wraps(f) async def wrapped_method(*args: P.args, **kwargs: P.kwargs) -> T: - try: - return await f(*args, **kwargs) - except Exception as exception: - attempts = retries if retries is not None else settings.get_retries_number() - if attempts <= 1 or not is_retriable(exception): - _log_and_decide(exception) - raise - first_error = exception - - @functools.wraps(f) - async def replay_first_error_then_call(*args: P.args, **kwargs: P.kwargs) -> T: - nonlocal first_error - if first_error is not None: - error, first_error = first_error, None - raise error - return await f(*args, **kwargs) - - retryer = tenacity.AsyncRetrying( - stop=tenacity.stop_after_attempt(attempts), - wait=tenacity.wait_exponential_jitter(), - retry=tenacity.retry_if_exception(_log_and_decide), - reraise=True, - before=tenacity.before_log(logger, logging.DEBUG), - ) - return await retryer(replay_first_error_then_call, *args, **kwargs) + attempt = 1 + while True: + try: + return await f(*args, **kwargs) + except Exception as exception: + attempts = retries if retries is not None else settings.get_retries_number() + if not _log_and_decide(exception) or attempt >= attempts: + raise + await asyncio.sleep(_backoff(attempt)) + attempt += 1 return wrapped_method diff --git a/pyproject.toml b/pyproject.toml index 844ba12..19aa175 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ classifiers = [ ] version = "0" dependencies = [ - "tenacity>=8.4.1", # 2.0.31 is the first release that tolerates the __static_attributes__ and __firstlineno__ # attributes Python 3.13 added; below it SQLAlchemy's TypingOnly check fails at import. # 2.0.20, not 2.0.18: advanced-alchemy requires it, so nothing here can exercise 2.0.18. diff --git a/tests/test_retry.py b/tests/test_retry.py index bb3ad18..1e7beda 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -1,11 +1,9 @@ import asyncio import typing -from unittest import mock import asyncpg import pytest import sqlalchemy -import tenacity from sqlalchemy.exc import DBAPIError from sqlalchemy.ext import asyncio as sa_async @@ -20,7 +18,7 @@ def _retriable_error() -> DBAPIError: def _record_backoff(monkeypatch: pytest.MonkeyPatch) -> list[float]: - """Replace the retryer's sleep with a recorder, so attempt counts cost no wall time.""" + """Replace the retry loop's sleep with a recorder, so attempt counts cost no wall time.""" waits: list[float] = [] async def _record(delay: float) -> None: @@ -121,17 +119,16 @@ async def _always_fails() -> None: await _always_fails() assert attempts == int(configured) - assert waits, "asyncio.sleep is no longer the retryer's sleep; this test would wait out real backoff" + assert waits, "asyncio.sleep is no longer the retry loop's sleep; this test would wait out real backoff" async def test_a_retry_never_re_runs_the_body_immediately(monkeypatch: pytest.MonkeyPatch) -> None: """INVARIANT: every retry is separated from the attempt before it by a non-zero wait. - Broken by dropping ``wait=tenacity.wait_exponential_jitter()`` from the retryer, or swapping it - for ``wait_none()`` -- a change that keeps every attempt-count assertion in this file green, - because the counts do not depend on the delay. Waiting is the reason to reach for tenacity at - all rather than a bare loop: both retriable classes are failures of an overloaded or unreachable - server, and re-running instantly turns one serialization conflict into a spin against the same + Broken by dropping the ``asyncio.sleep`` from the retry loop, or making ``_backoff`` return + ``0`` -- a change that keeps every attempt-count assertion in this file green, because the + counts do not depend on the delay. Both retriable classes are failures of an overloaded or + unreachable server, and re-running instantly turns one serialization conflict into a spin against the same row and one refused connection into a flood at a node that is already down. Tuning the curve -- the initial delay, the base, how much jitter -- stays within the invariant; removing the wait does not. @@ -212,25 +209,6 @@ async def select_one() -> int | None: assert attempts == 2 # noqa: PLR2004 # the closed connection, then a fresh one -async def test_a_first_attempt_that_succeeds_never_enters_tenacity(monkeypatch: pytest.MonkeyPatch) -> None: - """INVARIANT: the success path is one direct call; tenacity is built only after a retriable error. - - Broken by going back to building ``tenacity.AsyncRetrying`` before the first attempt. Every - behavioural test still passes then, because the retryer is correct, only slow: its attempt loop - costs about 20 µs per call, more than ``pool_pre_ping`` costs against a local database, and - almost every call succeeds first time (#71). - """ - retryer_class: typing.Final = mock.Mock(wraps=tenacity.AsyncRetrying) - monkeypatch.setattr(tenacity, "AsyncRetrying", retryer_class) - - @postgres_retry - async def _succeeds() -> str: - return "ok" - - assert await _succeeds() == "ok" - retryer_class.assert_not_called() - - async def test_a_retriable_first_failure_waits_once_then_succeeds(monkeypatch: pytest.MonkeyPatch) -> None: waits: typing.Final = _record_backoff(monkeypatch) attempts = 0