diff --git a/CHANGELOG.md b/CHANGELOG.md index a4ddbe7..598a74a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Version history +## 0.20.2 + +- Add a `max_rss` config flag and `--max-rss` CLI option to gracefully recycle workers + whose current RSS exceeds a configured MiB limit. + ## 0.20.1 - Drain in-flight requests until `graceful_timeout` on shutdown. diff --git a/README.md b/README.md index d3af8d0..9590813 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,18 @@ anyio.run(serve, app, Config()) See Hypercorn's [documentation](https://hypercorn.readthedocs.io/en/latest/how_to_guides/api_usage.html) for more details. + +## Worker recycling + +Workers can be recycled after a fixed number of requests with `--max-requests`, or after their +current resident set size exceeds a limit with `--max-rss`: + +```bash +anycorn --workers 4 --max-rss 512 module:app +``` + +`max_rss` is specified in MiB and is disabled by default. Anycorn samples the worker RSS +periodically and uses the same graceful shutdown path as request-count recycling, so in-flight +requests can drain until `graceful_timeout` before the master process respawns the worker. When +`--workers 0` is used, no master process exists to respawn the worker; exceeding `max_rss` stops +that single worker, matching `max_requests` behavior. diff --git a/pyproject.toml b/pyproject.toml index e5aded0..b444401 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ "typing_extensions; python_version<'3.11'", "wsproto >=0.14.0", "rich-click >=1.8.3,<2.0.0", + "psutil>=7.2.2", ] [project.optional-dependencies] diff --git a/src/anycorn/__main__.py b/src/anycorn/__main__.py index 59a4637..6ff6e12 100644 --- a/src/anycorn/__main__.py +++ b/src/anycorn/__main__.py @@ -102,6 +102,11 @@ def _load_config(config_path: str | None) -> Config: help="Maximum number of requests a worker will process before restarting", type=int, ) +@click.option( + "--max-rss", + help="Maximum RSS in MiB a worker may use before restarting", + type=int, +) @click.option( "--max-requests-jitter", help="This jitter causes the max-requests per worker to be " @@ -247,6 +252,7 @@ def main( # noqa: C901 PLR0913 PLR0912 PLR0915 graceful_timeout: int | None, read_timeout: int | None, max_requests: int | None, + max_rss: int | None, max_requests_jitter: int | None, group: int | None, worker_class: str | None, @@ -312,6 +318,8 @@ def main( # noqa: C901 PLR0913 PLR0912 PLR0915 cfg.logconfig = log_config if max_requests is not None: cfg.max_requests = max_requests + if max_rss is not None: + cfg.max_rss = max_rss if max_requests_jitter is not None: cfg.max_requests_jitter = max_requests_jitter if pid is not None: diff --git a/src/anycorn/config.py b/src/anycorn/config.py index 057cf92..889183e 100644 --- a/src/anycorn/config.py +++ b/src/anycorn/config.py @@ -139,6 +139,7 @@ def __init__(self) -> None: logger_class = Logger loglevel: str = "INFO" max_app_queue_size: int = 10 + max_rss: int | None = None max_requests: int | None = None max_requests_jitter: int = 0 pid_path: str | None = None diff --git a/src/anycorn/run.py b/src/anycorn/run.py index 6ed0a8c..9e92e93 100644 --- a/src/anycorn/run.py +++ b/src/anycorn/run.py @@ -18,6 +18,7 @@ import anyio import anyio.abc import anyio.streams.tls +import psutil import sniffio from .datagram import wrap_datagram_socket @@ -39,6 +40,9 @@ ) from .worker_context import WorkerContext +MAX_RSS_CHECK_INTERVAL = 30 +_BYTES_PER_MIB = 1024 * 1024 + if TYPE_CHECKING: from collections.abc import Awaitable, Callable from multiprocessing.context import BaseContext @@ -51,6 +55,34 @@ from exceptiongroup import BaseExceptionGroup +def _read_current_rss() -> int: + """Return this process's current RSS in bytes.""" + return psutil.Process().memory_info().rss + + +async def _watch_max_rss( + context: WorkerContext, + config: Config, + read_current_rss: Callable[[], int] | None = None, +) -> None: + if config.max_rss is None: + return + + read_current_rss = _read_current_rss if read_current_rss is None else read_current_rss + max_rss = config.max_rss * _BYTES_PER_MIB + while True: + await context.sleep(MAX_RSS_CHECK_INTERVAL) + current_rss = read_current_rss() + if current_rss > max_rss: + await config.log.info( + "Worker RSS %.1f MiB exceeded max_rss %d MiB; recycling worker", + current_rss / _BYTES_PER_MIB, + config.max_rss, + ) + await context.terminate.set() + return + + def run(config: Config) -> int: # noqa: C901, PLR0912, PLR0915 """Start the server, blocking until it exits, and return an exit code.""" if config.pid_path is not None: @@ -315,6 +347,7 @@ async def worker_serve( # noqa: C901, PLR0912, PLR0915 if shutdown_trigger is not None: tg.start_soon(raise_shutdown, shutdown_trigger, context.terminated.set) tg.start_soon(raise_shutdown, context.terminate.wait, context.terminated.set) + tg.start_soon(_watch_max_rss, context, config) for udp_server in udp_servers: await tg.start(udp_server.run) diff --git a/tests/test___main__.py b/tests/test___main__.py index b3b9bbb..8684af8 100644 --- a/tests/test___main__.py +++ b/tests/test___main__.py @@ -52,6 +52,7 @@ def test_load_config(monkeypatch: MonkeyPatch) -> None: ("--ciphers", "DHE-RSA-AES128-SHA", "ciphers"), ("--keep-alive", 20, "keep_alive_timeout"), ("--keyfile", "/path", "keyfile"), + ("--max-rss", 128, "max_rss"), ("--pid", "/path", "pid_path"), ("--root-path", "/path", "root_path"), ("--workers", 2, "workers"), diff --git a/tests/test_config.py b/tests/test_config.py index a62c14c..fd79353 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -20,6 +20,7 @@ access_log_format = "bob" h11_max_incomplete_size = 4 +max_rss = 128 # The reuse option _create_sockets sets is platform-specific: SO_EXCLUSIVEADDRUSE on # Windows (so a second server on a busy port fails), SO_REUSEADDR elsewhere. getattr @@ -64,6 +65,18 @@ def test_config_from_toml() -> None: _check_standard_config(config) +def test_config_from_toml_sets_max_rss(tmp_path: Path) -> None: + path = tmp_path / "config.toml" + path.write_text(f"max_rss = {max_rss}\n") + config = Config.from_toml(path) + assert config.max_rss == max_rss + + +def test_config_from_mapping_sets_max_rss() -> None: + config = Config.from_mapping(max_rss=max_rss) + assert config.max_rss == max_rss + + def test_create_ssl_context() -> None: path = str(Path(__file__).parent / "assets/config_ssl.py") config = Config.from_pyfile(path) diff --git a/tests/test_run.py b/tests/test_run.py index ced1e4d..c8f86df 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -140,6 +140,44 @@ async def run( assert observed == [True] +@pytest.mark.anyio +async def test_watch_max_rss_terminates_worker_when_over_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + config = Config() + config.max_rss = 1 + context = WorkerContext(None) + logged: list[tuple[str, tuple[object, ...]]] = [] + + async def info(message: str, *args: object) -> None: + logged.append((message, args)) + + monkeypatch.setattr(config.log, "info", info) + monkeypatch.setattr(anycorn.run, "MAX_RSS_CHECK_INTERVAL", 0) + + await anycorn.run._watch_max_rss(context, config, lambda: 2 * 1024 * 1024) + + assert context.terminate.is_set() + assert logged == [ + ( + "Worker RSS %.1f MiB exceeded max_rss %d MiB; recycling worker", + (2.0, 1), + ) + ] + + +@pytest.mark.anyio +async def test_worker_serve_terminates_when_over_max_rss(monkeypatch: pytest.MonkeyPatch) -> None: + config = Config() + config.bind = ["127.0.0.1:0"] + config.max_rss = 1 + monkeypatch.setattr(anycorn.run, "MAX_RSS_CHECK_INTERVAL", 0) + monkeypatch.setattr(anycorn.run, "_read_current_rss", lambda: 2 * 1024 * 1024) + + with anyio.fail_after(5): + await worker_serve(wrap_app(app, config.wsgi_max_body_size, None), config) + + @pytest.mark.anyio async def test_udp_server_serialises_concurrent_sends() -> None: """QUIC sends from several tasks at once must not collide on the socket. diff --git a/uv.lock b/uv.lock index 6d123c1..6975508 100644 --- a/uv.lock +++ b/uv.lock @@ -41,6 +41,7 @@ dependencies = [ { name = "h2" }, { name = "hpack" }, { name = "priority" }, + { name = "psutil" }, { name = "rich-click" }, { name = "sniffio" }, { name = "tomli", marker = "python_full_version < '3.11'" }, @@ -84,6 +85,7 @@ requires-dist = [ { name = "h2", specifier = ">=3.1.0" }, { name = "hpack" }, { name = "priority" }, + { name = "psutil", specifier = ">=7.2.2" }, { name = "rich-click", specifier = ">=1.8.3,<2.0.0" }, { name = "sniffio", specifier = ">=1.1,<2.0" }, { name = "tomli", marker = "python_full_version < '3.11'" }, @@ -319,7 +321,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -482,6 +484,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/5f/82c8074f7e84978129347c2c6ec8b6c59f3584ff1a20bc3c940a3e061790/priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa", size = 8946, upload-time = "2021-06-27T10:15:03.856Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pyasn1" version = "0.6.1"