diff --git a/DESIGN.md b/DESIGN.md index 19d4cea..f047c48 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -314,12 +314,15 @@ language changes — this is a property of the emitted code, and the recursive d only way to write it. The **async form** is the same rewrite one wrapper in. `let f a = async { … }` lowers to -`def f(a): async def g(): …; return g()`, and its self tail call is `return await f(…)` inside `g` — +`def f(a): async def g(): …; return _pf_Async(g)`, and its self tail call is `return await f(…)` +inside `g` — each message of an agent loop then awaits a fresh coroutine *on the same stack*, so CPython's limit is the loop's lifetime cap (measured: ~1,000 messages), which is fatal for exactly the thing an -agent is written to be: long-lived. When a body ends in that wrapper shape, the awaited tail calls -rebind the outer parameters (they are `g`'s closure, so a `nonlocal` declaration makes them -writable) and go round a `while True:` inside the one coroutine. For the tail call to be *visible*, +agent is written to be: long-lived. When a body ends in that wrapper shape, `g` takes the +parameters as its own (`async def g(a): while True: …`, returned as `_pf_Async(g, a)`), and the +awaited tail calls rebind them and go round the loop inside the one coroutine. Because an `Async` is +cold (§8), every await of `f(…)` starts again from the arguments it was built with, and no +`nonlocal` is needed. For the tail call to be *visible*, `return!` of a `match`/`if` in an `async { }` block lowers with a return per arm (`return await loop(inbox, n)` in the arm) rather than assigning a temp and awaiting it once — the same descent a sync function's tail `match` gets. The preconditions below apply to `g`'s body unchanged; the @@ -438,11 +441,11 @@ from their inferred types where Python can spell them: `int`, `float`, `bool`, ` record's class, and a union alias per user sum type (`Expr = Num | Add | Mul | Neg`, emitted after its classes). A type variable, a function type or an async result stays unannotated, so the annotations never claim more than the checker proved, and so does a `unit` result, since Pyfun -returns a unit call's value (`return f(x)`), which mypy rejects under `-> None`. A parameter -that a nested function rebinds through `nonlocal` (an async self tail call's, §5.4) stays -unannotated too: mypyc copies an annotated `int` parameter into the closure environment without -boxing it, which gcc rejects and an MSVC build turns into a segfault (python/mypy#22221). Data -class fields follow suit: a field declared as a user sum type or record names it (`_0: "Expr"`, +returns a unit call's value (`return f(x)`), which mypy rejects under `-> None`. Pyfun never +rebinds a parameter through `nonlocal` (the async self tail call loops over the coroutine's own +parameters, §5.4), which matters to mypyc: it copies an annotated `int` parameter into a closure +environment without boxing it, which gcc rejects and an MSVC build turns into a segfault +(python/mypy#22221). Data class fields follow suit: a field declared as a user sum type or record names it (`_0: "Expr"`, a string because the alias comes later), and any other field without a Python class of its own is `typing.Any`, since mypyc cannot build a dataclass with an `object` field. The output is written for mypy's newer redefinition rule (`--allow-redefinition-new --local-partial-types`): a @@ -840,8 +843,30 @@ lowers to an emitted `_pf_fmt_*` helper wrapping `format(x, spec)` (the spec a n from the checked `int`) or `str.rjust`/`ljust`. Dates are deferred (no date type; they would need a Python `datetime` `extern`). -**Async combinators — the `Async` module.** `Async a` is the type of a coroutine and `async { }` -builds one; the module is the rest of what F#'s `Async` offers that `asyncio` can supply, typed once +**Cold async.** An `Async a` is **cold**, as F#'s `Async` is: building one runs nothing, and every +await is a fresh run, so the same value can be awaited twice, retried, or passed to +`Async.parallel` twice and run twice. A Python coroutine is single-shot (a second `await` raises +`RuntimeError: cannot reuse already awaited coroutine`), so the value is not one. It is a small +emitted `_pf_Async(fn, *args)` whose `__await__` calls `fn(*args)` afresh: `let job = async { … }` +lowers to `job = _pf_Async(_pf_fn0)`, and a saturated call of an `extern` whose result is `Async` +to `_pf_Async(f, a, b)` (function and arguments evaluated at once; pinned keywords ride a +`functools.partial`). Awaiting one where it is built prints as the direct call (`await +inbox.get()`). Where Python insists on a real coroutine, a value passes through `_pf_run(x)` (`async +def _pf_run(a): return await a`), a fresh coroutine of any awaitable: an `extern` parameter typed +`Async a`, a callback parameter returning `Async` (wrapped so each call Python makes returns one: +`asyncio.start_server` schedules a handler's result only if it is a coroutine, and would otherwise +drop it silently), a `unit -> Async a` thunk, and every element `Async.parallel`/`race` start. In a +project the exporting module's binding for such an extern does the adapting, since a dependent +knows only its arity, and a bare or partially applied reference is the same adapting lambda +(`lambda p0: _pf_Async(asyncio.sleep, p0)`, under a `functools.partial` for supplied arguments), so +`urls |> List.map fetch` builds cold values too. Python code handed a Pyfun `Async` can `await` it +any number of times, or call it for a fresh coroutine, as it would a zero-argument `async def` +(`asyncio.run(job())`): `_pf_Async.__call__` is `async def __call__(self): return await self`. +Python 3.14's `asyncio.run` accepts any awaitable, but 3.12 and 3.13 want a coroutine, so code that +must run on them calls the value. + +**Async combinators — the `Async` module.** `async { }` builds an `Async a`; the module is the rest +of what F#'s `Async` offers that `asyncio` can supply, typed once (single source of truth `types::ASYNC_PRELUDE` + `seed_async_prelude`): `Async.sleep secs`, `Async.timeout secs c : Async (Option a)` (`asyncio.wait_for`, `None` on the deadline), `Async.toThread thunk` (a blocking call off the event loop, `asyncio.to_thread`), `Async.parallel @@ -852,9 +877,10 @@ partial member) and `Async.catch c : Async (Result a Exception)`, the await-time catches at *call* time, and an async extern does not throw when called, it returns a coroutine that throws when awaited, so `try` would wrap the coroutine and the `TimeoutError` would escape at the `let!`. `catch` catches `Exception`, not `BaseException`, so `asyncio.CancelledError` passes through -and structured cancellation keeps working. Building any of these is pure (a coroutine is a value); +and structured cancellation keeps working. Building any of these is pure (an `Async` is a value); the `async` effect is attributed where the value is awaited, except `toThread`, whose arrow carries -the thunk's effect. `sleep` is `asyncio.sleep` itself; the rest are emitted `_pf_async_*` helpers, +the thunk's effect. Each is an emitted helper returning a `_pf_Async` (a fully applied +`Async.sleep secs` builds `_pf_Async(asyncio.sleep, secs)` directly), because `gather(*xs)` spreads a list the extern syntax cannot, `wait` returns task sets, and two of them build `Option`/`Result` values. **Deliberately absent:** `Async.start` and `Async.cancel`. A task should only start inside a scope that will join or cancel it, which is the structured-concurrency diff --git a/INTERNALS.md b/INTERNALS.md index 0fbaea2..63b881b 100644 --- a/INTERNALS.md +++ b/INTERNALS.md @@ -417,9 +417,7 @@ A `Lowerer.native` flag, set by `lowering::lower_native_typed` (single file, cal `_X: X = X()`, and the first assignment of a top-level value bound once. - **Comparisons and closures.** `native_classes` sets `PyStmt::ClassDef::native`, so an ordered variant emits `__eq__` and all four orderings instead of `@total_ordering` (mypyc compiled `==` as - identity on a class with `__lt__` alone). After `annotate_functions`, `unannotate_nonlocal_params` - clears the annotation of any `TypedFuncDef` parameter a nested function names in a `nonlocal` - (python/mypy#22221). + identity on a class with `__lt__` alone). - **Discarded matches.** `Lowerer::push_discarded` emits a statement-position expression. In native mode, when its value is a temp set by an `if` ladder, `discard_temp` turns each assignment of the temp into a bare evaluation, since mypyc rejects a local whose every assignment is a unit @@ -608,6 +606,22 @@ top-level evaluation's performed effect (value bindings, expressions, module val `Task.scope`/`Task.start` to `_pf_task_scope`/`_pf_task_start` (`async_prelude`) and `note_task_scope_var` defines or imports `_pf_scope`. +**Cold async (DESIGN §8).** `lower_async` returns `Lowerer::cold_async([g])`, a +`_pf_Async(g)` call, and records `_pf_Async` in `needed_async_helpers`; `async_prelude` emits the +class as a `PyStmt::Class` (the plain-class IR node) and adds `_pf_Async`/`_pf_run` whenever a helper +that needs them is used. Every `Async`/`Task` helper is a plain `def` returning `_pf_Async(run)` +around an inner `async def run()`. At the boundary, `Lowerer::new` records each extern's +`async_param_slots` (`AsyncSlot::Value`/`Callback`/`Thunk`) and whether it `returns_async`; +`wrap_async_args` runs after `wrap_thunk_args` at every extern call site, and `lower_application` +wraps a saturated `Async`-returning extern call with `cold_extern_result` (`_pf_Async(f, *args)`, +or a `functools.partial` for pinned keywords). In a project, `async_extern_binding` gives such an +extern an adapting lambda, since a dependent knows only its arity; `lower_var_value` uses the same +lambda for a bare reference, and `lower_application` a `functools.partial` over it for a partial +application. `_pf_Async.__call__` gives Python code a fresh coroutine (`asyncio.run(job())`). The emitter prints +`await _pf_Async(f, *args)` as `await f(*args)`. The async self-tail-call rewrite +(`rewrite_async_wrapper`) matches `return _pf_Async(g)`, gives `g` the parameters and returns +`_pf_Async(g, *params)`. + ## Solving effect equations (`Infer::unify_eff`) — implements DESIGN §4 An `Effect` is a set of concrete labels plus a set of effect variables, so unification is over an diff --git a/ROADMAP.md b/ROADMAP.md index 090f81b..d716c0c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -525,9 +525,9 @@ map_build 1.64× vs hand-written. `tests/native_parity.py` (the `native` CI job) diffs every runnable lesson, the offline examples and the programs in `tests/native/` against `pyfun run`, so a native-only miscompile fails the PR. Two it found are fixed: `==` on a compared sum type (explicit comparison methods, §5.6) and an - async self tail call, whose parameters a coroutine rebinds through `nonlocal`. mypyc copied such - a parameter into the closure unboxed when it was annotated `int` (python/mypy#22221), so those - parameters stay unannotated (#173). + async self tail call, whose parameters a coroutine rebound through `nonlocal`, which mypyc copied + into the closure unboxed when they were annotated `int` (python/mypy#22221, #173). Since `Async` + went cold, that loop runs over the coroutine's own parameters and needs no `nonlocal` at all. The original reasoning, kept for the record: the checker knows every binding's inferred type, so the emitter could produce fully annotated Python whose annotations cannot lie, then compile it with mypyc into a C extension — native speed diff --git a/docs/src/learn/18-async.md b/docs/src/learn/18-async.md index e422404..4c14de4 100644 --- a/docs/src/learn/18-async.md +++ b/docs/src/learn/18-async.md @@ -22,28 +22,63 @@ print (runAsync fetchScore) ``` The key idea carries straight over from Python. An `async { }` block does not run when you write it. -It builds a coroutine, exactly like calling an `async def` in Python hands you an awaitable rather -than a result. Nothing happens until something drives that coroutine. Here `let! x = async { return -20 }` awaits an inner block and binds `20` to `x`, then `return x + 1` produces the final `21`, but -only once the coroutine is run. +It describes work to do later, much as calling an `async def` in Python hands you something to await +rather than a result. Nothing happens until something awaits it. Here `let! x = async { return 20 }` +awaits an inner block and binds `20` to `x`, then `return x + 1` produces the final `21`, but only +once the whole block is run. The emitted Python shows the mapping directly. Each `async { }` becomes a nested `async def`, each -`let!` becomes an `await`, and the whole value is a coroutine object: +`let!` becomes an `await`, and the value itself is a small `_pf_Async` that remembers which function +to run. (Two short helpers, `_pf_Async` and `_pf_run`, sit at the top of the file and are left out +here.) ```python -import asyncio async def _pf_fn1(): async def _pf_fn0(): return 20 x = await _pf_fn0() return x + 1 -fetchScore = _pf_fn1() -print(asyncio.run(fetchScore)) +fetchScore = _pf_Async(_pf_fn1) +print(asyncio.run(_pf_run(fetchScore))) ``` +## Awaiting a value twice + +Here is one place where Pyfun goes further than Python. Try awaiting the same value twice: + +```pyfun +extern runAsync: Async a -> a = asyncio.run + +let roll = + async { + print "rolling" + return 6 + } + +let twice = + async { + let! a = roll + let! b = roll + return a + b + } + +print (runAsync twice) +``` + +```console +rolling +rolling +12 +``` + +The block runs twice, once for each `let!`. A Python coroutine can only be awaited once, and a second +`await` of it raises an error. A Pyfun `Async` value can be awaited as often as you like, and each +await runs it again from the start. F# calls this a *cold* async. It means you can keep an `Async` in +a variable, pass it to a function that retries it, or put the same one in a list twice. + ## Running one at the top level -A coroutine has to be awaited somewhere, and at the top level there is no enclosing `async` +An `Async` value has to be awaited somewhere, and at the top level there is no enclosing `async` function to await it in. Python solves this with `asyncio.run(main())`, and Pyfun makes the same move through `extern` (lesson 12). The line @@ -52,7 +87,8 @@ extern runAsync: Async a -> a = asyncio.run ``` names Python's `asyncio.run` and gives it the Pyfun type `Async a -> a`: hand it an `Async a` and it -drives the coroutine to completion and returns the `a`. So `fetchScore |> runAsync` turns the +runs it to completion and returns the `a`. `asyncio.run` wants a Python coroutine, so Pyfun hands it +a fresh one made from the value, which is the `_pf_run(fetchScore)` you saw in the output. So `fetchScore |> runAsync` turns the `Async int` into a plain `int` you can `print`. ## Async is an inferred effect @@ -82,7 +118,9 @@ purity cannot be claimed over it. An async Python function returns a coroutine when you call it, and nothing happens until that coroutine is awaited. The Pyfun type says exactly that: the result is `Async a`, and `let!` (or -`do!` when there is no value) inside an `async { }` block is where the await happens. +`do!` when there is no value) inside an `async { }` block is where the await happens. Pyfun saves +the call itself for the await, so awaiting the result twice calls the Python function twice, just as +it runs an `async { }` block twice. ```pyfun extern runAsync: Async a -> a = asyncio.run @@ -226,7 +264,7 @@ effect system tracks it for you. Fill the hole so `combined` awaits two async blocks and returns their sum. The hole has type `int` (the value the second block returns), so any integer literal works. The runner bridge over -`asyncio.run` is already in place, and `runAsync combined` drives the coroutine to a value. +`asyncio.run` is already in place, and `runAsync combined` runs it to a value. ```pyfun extern runAsync: Async a -> a = asyncio.run @@ -266,5 +304,5 @@ print (runAsync combined) ``` `let!` awaits each inner block and binds its result, so `a` is `10` and `b` is `20`, and `return a + -b` produces `30` once `runAsync` drives the coroutine. +b` produces `30` once `runAsync` runs the block. diff --git a/docs/src/notebooks/18-async.ipynb b/docs/src/notebooks/18-async.ipynb index 9b093ec..c15161e 100644 --- a/docs/src/notebooks/18-async.ipynb +++ b/docs/src/notebooks/18-async.ipynb @@ -48,32 +48,80 @@ "```\n", "\n", "The key idea carries straight over from Python. An `async { }` block does not run when you write it.\n", - "It builds a coroutine, exactly like calling an `async def` in Python hands you an awaitable rather\n", - "than a result. Nothing happens until something drives that coroutine. Here `let! x = async { return\n", - "20 }` awaits an inner block and binds `20` to `x`, then `return x + 1` produces the final `21`, but\n", - "only once the coroutine is run.\n", + "It describes work to do later, much as calling an `async def` in Python hands you something to await\n", + "rather than a result. Nothing happens until something awaits it. Here `let! x = async { return 20 }`\n", + "awaits an inner block and binds `20` to `x`, then `return x + 1` produces the final `21`, but only\n", + "once the whole block is run.\n", "\n", "The emitted Python shows the mapping directly. Each `async { }` becomes a nested `async def`, each\n", - "`let!` becomes an `await`, and the whole value is a coroutine object:\n", + "`let!` becomes an `await`, and the value itself is a small `_pf_Async` that remembers which function\n", + "to run. (Two short helpers, `_pf_Async` and `_pf_run`, sit at the top of the file and are left out\n", + "here.)\n", "\n", "```python\n", - "import asyncio\n", "async def _pf_fn1():\n", " async def _pf_fn0():\n", " return 20\n", " x = await _pf_fn0()\n", " return x + 1\n", - "fetchScore = _pf_fn1()\n", - "print(asyncio.run(fetchScore))\n", + "fetchScore = _pf_Async(_pf_fn1)\n", + "print(asyncio.run(_pf_run(fetchScore)))\n", "```\n", "\n", + "## Awaiting a value twice\n", + "\n", + "Here is one place where Pyfun goes further than Python. Try awaiting the same value twice:" + ], + "id": "18-003" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + ":reset\n", + "extern runAsync: Async a -> a = asyncio.run\n", + "\n", + "let roll =\n", + " async {\n", + " print \"rolling\"\n", + " return 6\n", + " }\n", + "\n", + "let twice =\n", + " async {\n", + " let! a = roll\n", + " let! b = roll\n", + " return a + b\n", + " }\n", + "\n", + "print (runAsync twice)" + ], + "id": "18-004" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```console\n", + "rolling\n", + "rolling\n", + "12\n", + "```\n", + "\n", + "The block runs twice, once for each `let!`. A Python coroutine can only be awaited once, and a second\n", + "`await` of it raises an error. A Pyfun `Async` value can be awaited as often as you like, and each\n", + "await runs it again from the start. F# calls this a *cold* async. It means you can keep an `Async` in\n", + "a variable, pass it to a function that retries it, or put the same one in a list twice.\n", + "\n", "## Running one at the top level\n", "\n", - "A coroutine has to be awaited somewhere, and at the top level there is no enclosing `async`\n", + "An `Async` value has to be awaited somewhere, and at the top level there is no enclosing `async`\n", "function to await it in. Python solves this with `asyncio.run(main())`, and Pyfun makes the same\n", "move through `extern` (lesson 12). The line" ], - "id": "18-003" + "id": "18-005" }, { "cell_type": "code", @@ -84,14 +132,15 @@ ":reset\n", "extern runAsync: Async a -> a = asyncio.run" ], - "id": "18-004" + "id": "18-006" }, { "cell_type": "markdown", "metadata": {}, "source": [ "names Python's `asyncio.run` and gives it the Pyfun type `Async a -> a`: hand it an `Async a` and it\n", - "drives the coroutine to completion and returns the `a`. So `fetchScore |> runAsync` turns the\n", + "runs it to completion and returns the `a`. `asyncio.run` wants a Python coroutine, so Pyfun hands it\n", + "a fresh one made from the value, which is the `_pf_run(fetchScore)` you saw in the output. So `fetchScore |> runAsync` turns the\n", "`Async int` into a plain `int` you can `print`.\n", "\n", "## Async is an inferred effect\n", @@ -100,7 +149,7 @@ "of those effect labels, and an `async { }` block performs it. So a `let pure` body that builds an\n", "async block is a compile error, the same way a `let pure` that prints is:" ], - "id": "18-005" + "id": "18-007" }, { "cell_type": "code", @@ -110,7 +159,7 @@ "source": [ "let pure grab x = async { return x + 1 }" ], - "id": "18-006" + "id": "18-008" }, { "cell_type": "markdown", @@ -133,9 +182,11 @@ "\n", "An async Python function returns a coroutine when you call it, and nothing happens until that\n", "coroutine is awaited. The Pyfun type says exactly that: the result is `Async a`, and `let!` (or\n", - "`do!` when there is no value) inside an `async { }` block is where the await happens." + "`do!` when there is no value) inside an `async { }` block is where the await happens. Pyfun saves\n", + "the call itself for the await, so awaiting the result twice calls the Python function twice, just as\n", + "it runs an `async { }` block twice." ], - "id": "18-007" + "id": "18-009" }, { "cell_type": "code", @@ -155,7 +206,7 @@ "\n", "print (runAsync nap)" ], - "id": "18-008" + "id": "18-010" }, { "cell_type": "markdown", @@ -185,7 +236,7 @@ "not fail); `Async.race` takes the first to finish and cancels the rest; and `Async.toThread` runs a\n", "blocking function, `input` say, on a worker thread so the event loop stays free." ], - "id": "18-009" + "id": "18-011" }, { "cell_type": "code", @@ -217,7 +268,7 @@ "\n", "runAsync main" ], - "id": "18-010" + "id": "18-012" }, { "cell_type": "markdown", @@ -241,7 +292,7 @@ "scope waits for every task it started (or cancels them all if one fails). There is no `cancel` to\n", "forget." ], - "id": "18-011" + "id": "18-013" }, { "cell_type": "code", @@ -267,7 +318,7 @@ "\n", "print (runAsync session)" ], - "id": "18-012" + "id": "18-014" }, { "cell_type": "markdown", @@ -286,7 +337,7 @@ "tasks is fine to write (its type says `->{spawn}`), and calling it inside a scope's body is fine\n", "too. Starting a task anywhere else is a compile error:" ], - "id": "18-013" + "id": "18-015" }, { "cell_type": "code", @@ -296,7 +347,7 @@ "source": [ "let stray = Task.start (Async.sleep 1.0)" ], - "id": "18-014" + "id": "18-016" }, { "cell_type": "markdown", @@ -328,9 +379,9 @@ "\n", "Fill the hole so `combined` awaits two async blocks and returns their sum. The hole has type `int`\n", "(the value the second block returns), so any integer literal works. The runner bridge over\n", - "`asyncio.run` is already in place, and `runAsync combined` drives the coroutine to a value." + "`asyncio.run` is already in place, and `runAsync combined` runs it to a value." ], - "id": "18-015" + "id": "18-017" }, { "cell_type": "code", @@ -350,7 +401,7 @@ "\n", "print (runAsync combined)" ], - "id": "18-016" + "id": "18-018" }, { "cell_type": "markdown", @@ -381,10 +432,10 @@ "```\n", "\n", "`let!` awaits each inner block and binds its result, so `a` is `10` and `b` is `20`, and `return a +\n", - "b` produces `30` once `runAsync` drives the coroutine.\n", + "b` produces `30` once `runAsync` runs the block.\n", "" ], - "id": "18-017" + "id": "18-019" } ], "metadata": { diff --git a/src/lowering/mod.rs b/src/lowering/mod.rs index 7ad1db3..d7b3cf7 100644 --- a/src/lowering/mod.rs +++ b/src/lowering/mod.rs @@ -360,6 +360,12 @@ struct Lowerer { /// no arguments, so the call site wraps it as `lambda: f(None)` (`DESIGN.md` /// §6, issue #107). extern_thunk_slots: std::collections::HashMap>, + /// `extern` name → its parameters that carry a Pyfun `Async` into Python, and + /// how each crosses (`DESIGN.md` §8): Python receives a real coroutine. + extern_async_slots: std::collections::HashMap>, + /// The `extern`s whose result is `Async`: a saturated call is wrapped cold, + /// `_pf_Async(f, *args)`, so awaiting it twice calls Python twice. + extern_async_results: HashSet, /// `extern` name → its Python keyword arguments (literals already lowered to /// `PyExpr`, `...` slots left to be filled from the call), appended to every /// emitted call (`open(path, encoding="utf-8")`). Under-application routes @@ -551,6 +557,8 @@ impl Lowerer { let mut receiver_externs = std::collections::HashMap::new(); let mut extern_unit_slots = std::collections::HashMap::new(); let mut extern_thunk_slots = std::collections::HashMap::new(); + let mut extern_async_slots = std::collections::HashMap::new(); + let mut extern_async_results = HashSet::new(); let mut extern_kwargs: std::collections::HashMap> = std::collections::HashMap::new(); let mut user_defs = HashSet::new(); @@ -574,6 +582,13 @@ impl Lowerer { if !units.is_empty() { extern_unit_slots.insert(decl.name.clone(), units); } + let async_slots = async_param_slots(&decl.ty); + if !async_slots.is_empty() { + extern_async_slots.insert(decl.name.clone(), async_slots); + } + if returns_async(&decl.ty) { + extern_async_results.insert(decl.name.clone()); + } let slots = thunk_param_slots(&decl.ty); if !slots.is_empty() { extern_thunk_slots.insert(decl.name.clone(), slots); @@ -754,6 +769,8 @@ impl Lowerer { receiver_externs, extern_unit_slots, extern_thunk_slots, + extern_async_slots, + extern_async_results, extern_kwargs, needed_imports: BTreeSet::new(), user_defs, @@ -812,11 +829,7 @@ impl Lowerer { return lowered .map(any_for_object_fields) .map(native_classes) - .map(|py| self.annotate_functions(module, py)) - .map(|mut py| { - unannotate_nonlocal_params(&mut py.body); - py - }); + .map(|py| self.annotate_functions(module, py)); } lowered } @@ -1202,6 +1215,7 @@ impl Lowerer { ) } }; + let value = self.async_extern_binding(&decl.name, k, value); code.push(PyStmt::Assign { target: py_value_name(&decl.name), value, @@ -2175,7 +2189,7 @@ impl Lowerer { ExprKind::Str(s) => Ok((vec![], PyExpr::Str(s.clone()))), ExprKind::Bool(b) => Ok((vec![], PyExpr::Bool(*b))), ExprKind::Unit => Ok((vec![], PyExpr::NoneLit)), - ExprKind::Var(name) => Ok((vec![], self.lower_var(name, locals))), + ExprKind::Var(name) => Ok((vec![], self.lower_var_value(name, locals))), ExprKind::Binary { op, lhs, rhs } => { let (mut stmts, left) = self.lower_value(lhs, locals)?; @@ -2741,6 +2755,63 @@ impl Lowerer { fn lower_application(&mut self, expr: &Expr, locals: &HashSet) -> Lowered { let mut args_ast = Vec::new(); let head = flatten_app(expr, &mut args_ast); + // `Async.sleep secs` builds `_pf_Async(asyncio.sleep, secs)` directly, so an + // immediate `do!` reads `await asyncio.sleep(secs)`; a bare `Async.sleep` + // passed as a value keeps its helper. + if args_ast.len() == 1 + && crate::types::qualified_name(head).is_some_and(|q| q == "Async.sleep") + { + let (stmts, secs) = self.lower_value(args_ast[0], locals)?; + self.needed_imports.insert("asyncio".to_string()); + let sleep = PyExpr::Attribute { + value: Box::new(PyExpr::Name("asyncio".to_string())), + attr: "sleep".to_string(), + }; + return Ok((stmts, self.cold_async(vec![sleep, secs]))); + } + // A partial application of an `extern` that carries `Async` closes over + // its adapting lambda, so each later call still yields a cold value and + // gives Python real coroutines (`urls |> List.map (fetch session)`). + if let ExprKind::Var(name) = &head.kind + && self.is_async_extern(name, locals) + && let Some(&k) = self.arities.get(name) + && !args_ast.is_empty() + && args_ast.len() < k + { + let mut stmts = Vec::new(); + let mut args = Vec::with_capacity(args_ast.len()); + for arg in &args_ast { + let (arg_stmts, arg_val) = self.lower_value(arg, locals)?; + stmts.extend(arg_stmts); + args.push(arg_val); + } + let bare = self.lower_var(name, locals); + let adapter = self.async_extern_binding(name, k, bare); + self.needs_functools = true; + let partial = PyExpr::Call { + func: Box::new(PyExpr::Attribute { + value: Box::new(PyExpr::Name("functools".to_string())), + attr: "partial".to_string(), + }), + args: std::iter::once(adapter).chain(args).collect(), + }; + return Ok((stmts, partial)); + } + let (stmts, value) = self.lower_application_inner(expr, locals)?; + if let ExprKind::Var(name) = &head.kind + && !locals.contains(name) + && !self.user_defs.contains(name) + && self.extern_async_results.contains(name) + && self.arities.get(name) == Some(&args_ast.len()) + { + return Ok((stmts, self.cold_extern_result(value))); + } + Ok((stmts, value)) + } + + fn lower_application_inner(&mut self, expr: &Expr, locals: &HashSet) -> Lowered { + let mut args_ast = Vec::new(); + let head = flatten_app(expr, &mut args_ast); // Tier-1 in-place linear accumulation (`DESIGN.md` §5): a qualifying // fully-applied `Seq.fold`/`List.fold` inlines to a `for`-loop over a @@ -2820,6 +2891,7 @@ impl Lowerer { arg_vals.push(arg_val); } self.wrap_thunk_args(name, &mut arg_vals); + self.wrap_async_args(name, &mut arg_vals); // A bare (unapplied) reference becomes a receiver-taking lambda (which // pins any kwargs itself, `lambda r, a: r.write_text(a, encoding=…)`). if arg_vals.is_empty() { @@ -2919,6 +2991,7 @@ impl Lowerer { arg_vals.push(arg_val); } self.wrap_thunk_args(name, &mut arg_vals); + self.wrap_async_args(name, &mut arg_vals); let call = self.build_call_units(dotted_path(&target), k, &units, arg_vals, spec, &mut stmts); return Ok((stmts, call)); @@ -2945,6 +3018,7 @@ impl Lowerer { arg_vals.push(arg_val); } self.wrap_thunk_args(name, &mut arg_vals); + self.wrap_async_args(name, &mut arg_vals); let mut hoist = Vec::new(); let call = self.build_call_kw(dotted_path(&target), arity, arg_vals, kwargs, &mut hoist); @@ -3012,7 +3086,12 @@ impl Lowerer { _ => None, }; - let (mut stmts, head_val) = self.lower_value(head, locals)?; + // A variable head lowers bare: the call itself is adapted where it is + // built, so an `Async`-touching extern is not first wrapped as a value. + let (mut stmts, head_val) = match &head.kind { + ExprKind::Var(name) => (Vec::new(), self.lower_var(name, locals)), + _ => self.lower_value(head, locals)?, + }; let mut arg_vals = Vec::with_capacity(args_ast.len()); for arg in &args_ast { let (arg_stmts, arg_val) = self.lower_value(arg, locals)?; @@ -3024,6 +3103,7 @@ impl Lowerer { && !locals.contains(name) { self.wrap_thunk_args(name, &mut arg_vals); + self.wrap_async_args(name, &mut arg_vals); } Ok((stmts, self.build_call(head_val, arity, arg_vals))) @@ -3590,13 +3670,7 @@ impl Lowerer { // `asyncio.sleep` itself; the rest are `_pf_async_*` helpers, since // `gather(*xs)` spreads a list, `wait` returns task sets, and // `timeout`/`catch` build `Option`/`Result` values. - "Async.sleep" => { - self.needed_imports.insert("asyncio".to_string()); - PyExpr::Attribute { - value: Box::new(bare("asyncio")), - attr: "sleep".to_string(), - } - } + "Async.sleep" => asy(self, "_pf_async_sleep"), "Async.timeout" => { self.needs_option = true; asy(self, "_pf_async_timeout") @@ -5039,7 +5113,8 @@ impl Lowerer { out } - /// `async { ... }` → an `async def` returning a coroutine. + /// `async { ... }` → an `async def` wrapped in a cold `_pf_Async`: nothing runs + /// until it is awaited, and each await runs it afresh (`DESIGN.md` §8). fn lower_async(&mut self, items: &[CeItem], locals: &HashSet) -> Lowered { let mut body = Vec::new(); let mut locals = locals.clone(); @@ -5057,7 +5132,27 @@ impl Lowerer { body: full, is_async: true, }; - Ok((vec![def], call0(&name))) + Ok((vec![def], self.cold_async(vec![PyExpr::Name(name)]))) + } + + /// `_pf_Async(fn, *args)`: a cold async value whose every await calls + /// `fn(*args)` for a fresh run. Records that the runtime class is needed. + fn cold_async(&mut self, args: Vec) -> PyExpr { + self.needed_async_helpers.insert("_pf_Async"); + PyExpr::Call { + func: Box::new(PyExpr::Name("_pf_Async".to_string())), + args, + } + } + + /// `_pf_run(value)`: a fresh coroutine of any awaitable, for a Python API + /// that insists on a real one. Records that the helper is needed. + fn run_async(&mut self, value: PyExpr) -> PyExpr { + self.needed_async_helpers.insert("_pf_run"); + PyExpr::Call { + func: Box::new(PyExpr::Name("_pf_run".to_string())), + args: vec![value], + } } /// One level of `async` items into `body`; a `for` item nests a level inside @@ -5333,6 +5428,142 @@ impl Lowerer { /// Python with no arguments, so it becomes `lambda: f(None)`, and a literal /// `fun _ -> body` collapses to `lambda: body`. Only the slots the extern's /// declared type marks are touched; every other argument passes as it is. + /// Adapt the arguments of an `extern` call that carry a Pyfun `Async` into + /// Python (`DESIGN.md` §8). A Pyfun `Async` is a cold `_pf_Async`, but Python + /// APIs want a real coroutine: `create_task` rejects anything else, and + /// `asyncio.start_server` schedules a handler's result only if it is one, so + /// a handler returning `_pf_Async` would silently never run. An `Async` + /// argument passes as `_pf_run(x)`, and a callback returning `Async` is wrapped + /// so every call Python makes returns `_pf_run(...)`. + fn wrap_async_args(&mut self, name: &str, args: &mut [PyExpr]) { + let Some(slots) = self.extern_async_slots.get(name).cloned() else { + return; + }; + for (i, slot) in slots { + if i < args.len() { + let arg = std::mem::replace(&mut args[i], PyExpr::NoneLit); + args[i] = self.adapt_async_arg(slot, arg); + } + } + } + + /// One argument of [`Self::wrap_async_args`]. A thunk slot has already become + /// `lambda: body` through [`Self::wrap_thunk_args`]; its body is what runs. + fn adapt_async_arg(&mut self, slot: AsyncSlot, arg: PyExpr) -> PyExpr { + match slot { + AsyncSlot::Value => self.run_async(arg), + AsyncSlot::Thunk => match arg { + PyExpr::Lambda { params, body } if params.is_empty() => PyExpr::Lambda { + params, + body: Box::new(self.run_async(*body)), + }, + f => { + let call = PyExpr::Call { + func: Box::new(f), + args: vec![PyExpr::NoneLit], + }; + PyExpr::Lambda { + params: vec![], + body: Box::new(self.run_async(call)), + } + } + }, + AsyncSlot::Callback => { + let call = PyExpr::Call { + func: Box::new(arg), + args: vec![PyExpr::Starred(Box::new(PyExpr::Name("_pf_a".to_string())))], + }; + PyExpr::Lambda { + params: vec!["*_pf_a".to_string()], + body: Box::new(self.run_async(call)), + } + } + } + } + + /// A project's binding for an `extern` that carries `Async` across the + /// boundary. A dependent module calls the binding n-ary and knows only its + /// arity, so the binding itself adapts: `lambda p0, p1: _pf_Async(f, _pf_run(p0), + /// p1)` for an `Async` argument and result. Any other extern binds unchanged. + fn async_extern_binding(&mut self, name: &str, k: usize, value: PyExpr) -> PyExpr { + let slots = self + .extern_async_slots + .get(name) + .cloned() + .unwrap_or_default(); + let cold = self.extern_async_results.contains(name); + if k == 0 || (slots.is_empty() && !cold) { + return value; + } + let params: Vec = (0..k).map(|i| format!("_pf_p{i}")).collect(); + let mut args: Vec = params.iter().map(|p| PyExpr::Name(p.clone())).collect(); + for (i, slot) in slots { + if i < args.len() { + let arg = std::mem::replace(&mut args[i], PyExpr::NoneLit); + args[i] = self.adapt_async_arg(slot, arg); + } + } + let body = if cold { + self.cold_async(std::iter::once(value).chain(args).collect()) + } else { + PyExpr::Call { + func: Box::new(value), + args, + } + }; + PyExpr::Lambda { + params, + body: Box::new(body), + } + } + + /// Whether `name` here is an `extern` that carries `Async` across the boundary + /// (an `Async` result or parameter), rather than a local or user binding. + fn is_async_extern(&self, name: &str, locals: &HashSet) -> bool { + !locals.contains(name) + && !self.user_defs.contains(name) + && (self.extern_async_results.contains(name) + || self.extern_async_slots.contains_key(name)) + } + + /// A variable in value position. A bare `extern` that carries `Async` is its + /// adapting lambda (see [`Self::async_extern_binding`]), so `List.map fetch + /// urls` builds cold values and Python receives coroutines. + fn lower_var_value(&mut self, name: &str, locals: &HashSet) -> PyExpr { + let bare = self.lower_var(name, locals); + match self.arities.get(name).copied() { + Some(k) if self.is_async_extern(name, locals) => { + self.async_extern_binding(name, k, bare) + } + _ => bare, + } + } + + /// A saturated call of an `extern` whose result is `Async`, made cold: the + /// Python call runs on each await, with the function and arguments evaluated + /// now. Pinned keyword arguments ride a `functools.partial`. Any other shape + /// (a property read) keeps the awaitable Python returned. + fn cold_extern_result(&mut self, value: PyExpr) -> PyExpr { + match value { + PyExpr::Call { func, args } => { + self.cold_async(std::iter::once(*func).chain(args).collect()) + } + PyExpr::CallKw { func, args, kwargs } => { + self.needs_functools = true; + let partial = PyExpr::CallKw { + func: Box::new(PyExpr::Attribute { + value: Box::new(PyExpr::Name("functools".to_string())), + attr: "partial".to_string(), + }), + args: std::iter::once(*func).chain(args).collect(), + kwargs, + }; + self.cold_async(vec![partial]) + } + other => other, + } + } + fn wrap_thunk_args(&self, name: &str, args: &mut [PyExpr]) { let Some(slots) = self.extern_thunk_slots.get(name) else { return; @@ -6112,6 +6343,56 @@ fn unit_param_slots(ty: &TypeExpr) -> Vec { out } +/// How an `extern` parameter carries a Pyfun `Async` into Python (`DESIGN.md` §8). +#[derive(Clone, Copy, Debug, PartialEq)] +enum AsyncSlot { + /// An `Async a` argument: Python receives a fresh coroutine, `_pf_run(x)`. + Value, + /// A callback returning `Async` (`a -> Async b`): every call Python makes + /// returns a real coroutine, `lambda *_pf_a: _pf_run(f(*_pf_a))`. + Callback, + /// `unit -> Async a`: `lambda: _pf_run(f(None))`. + Thunk, +} + +/// `Async a`. +fn is_async_type(ty: &TypeExpr) -> bool { + matches!(ty, TypeExpr::Con(name, _, _) if name == "Async") +} + +/// Whether `ty`, after all its arrows, is `Async`: an extern's result, or a +/// callback parameter's. +fn returns_async(ty: &TypeExpr) -> bool { + let mut cur = ty; + while let TypeExpr::Fun(_, ret, _) = cur { + cur = ret; + } + is_async_type(cur) +} + +/// The positions (0-based, along the curried parameter list) of an extern's +/// parameters that carry a Pyfun `Async`, each with how it crosses. +fn async_param_slots(ty: &TypeExpr) -> Vec<(usize, AsyncSlot)> { + let mut out = Vec::new(); + let mut cur = ty; + let mut i = 0; + while let TypeExpr::Fun(domain, ret, _) = cur { + if is_async_type(domain) { + out.push((i, AsyncSlot::Value)); + } else if matches!(domain.as_ref(), TypeExpr::Fun(..)) && returns_async(domain) { + let slot = if is_unit_domain(domain) { + AsyncSlot::Thunk + } else { + AsyncSlot::Callback + }; + out.push((i, slot)); + } + i += 1; + cur = ret; + } + out +} + /// The positions (0-based, along the curried parameter list) of an extern's /// `unit -> a` parameters: callbacks Python calls with no arguments. fn thunk_param_slots(ty: &TypeExpr) -> Vec { @@ -6669,59 +6950,6 @@ fn native_classes(mut module: PyModule) -> PyModule { module } -/// Native mode: a parameter that a nested function rebinds through `nonlocal` -/// stays unannotated. mypyc copies an annotated `int` parameter into the -/// closure environment unboxed when the nested function sees the name as -/// another type, which gcc rejects and an MSVC build turns into a segfault -/// (python/mypy#22221). An async or `seq` self tail call rebinds its parameters -/// this way. A local the nested function rebinds is unaffected. -fn unannotate_nonlocal_params(stmts: &mut [PyStmt]) { - for stmt in stmts { - if let PyStmt::TypedFuncDef { params, body, .. } = stmt { - let mut rebound = HashSet::new(); - collect_nonlocals(body, &mut rebound); - // A function's own `nonlocal` names an enclosing binding, never one - // of its parameters (Python rejects that), so every name collected - // from its body is rebound by a function nested inside it. - for (name, ty) in params.iter_mut() { - if rebound.contains(name) { - *ty = None; - } - } - } - for block in child_blocks(stmt) { - unannotate_nonlocal_params(block); - } - } -} - -/// Every name a `nonlocal` declares anywhere in `stmts`, nested functions included. -fn collect_nonlocals(stmts: &mut [PyStmt], out: &mut HashSet) { - for stmt in stmts { - if let PyStmt::Nonlocal(names) = stmt { - out.extend(names.iter().cloned()); - } - for block in child_blocks(stmt) { - collect_nonlocals(block, out); - } - } -} - -/// The statement blocks directly inside `stmt`. -fn child_blocks(stmt: &mut PyStmt) -> Vec<&mut Vec> { - match stmt { - PyStmt::WhileTrue { body } - | PyStmt::For { body, .. } - | PyStmt::AsyncWith { body, .. } - | PyStmt::FuncDef { body, .. } - | PyStmt::TypedFuncDef { body, .. } => vec![body], - PyStmt::If { body, orelse, .. } => vec![body, orelse], - PyStmt::Try { body, handler, .. } => vec![body, handler], - PyStmt::Match { cases, .. } => cases.iter_mut().map(|c| &mut c.body).collect(), - _ => Vec::new(), - } -} - /// The Python spelling of each non-generic local user type for native /// annotations: a record is its class, and a sum type its union alias (which /// `annotate_functions` emits after the classes), unless a constructor shares @@ -7000,9 +7228,15 @@ fn combinator_prelude(used: &BTreeSet<&'static str>) -> Vec { /// primitives are strict — `_pf_dec_int` rejects a JSON bool (a Python `bool` is an /// `int` subclass) and `_pf_dec_float` accepts an int — so "parse, don't validate" /// actually validates. Built from the IR, emitted in sorted helper-name order. -/// The `Async`-module helpers referenced by the program (`DESIGN.md` §8.1): each -/// an `async def` over `asyncio`, except `toThread`, which returns the awaitable -/// `asyncio.to_thread` builds. `catch` deliberately catches `Exception`, not +/// The `Async`-module runtime referenced by the program (`DESIGN.md` §8). An +/// `Async a` is **cold**: a `_pf_Async` holds the coroutine function and its +/// arguments, and every `await` of it starts a fresh run, as F#'s `Async` does, +/// so one value can be awaited twice, retried or raced against itself. `_pf_run` +/// turns any awaitable into a real coroutine for the `asyncio` APIs that insist on +/// one (`create_task`, `gather`, `wait_for`), and gives each element of a list its +/// own run (`gather` would otherwise await a repeated value once). Each helper is a +/// plain function returning a `_pf_Async`, so building one is pure and nothing +/// runs until it is awaited. `catch` deliberately catches `Exception`, not /// `BaseException`, so `asyncio.CancelledError` passes through and structured /// cancellation keeps working. fn async_prelude(used: &BTreeSet<&'static str>, none_singleton: bool) -> Vec { @@ -7017,40 +7251,142 @@ fn async_prelude(used: &BTreeSet<&'static str>, none_singleton: bool) -> Vec| PyStmt::FuncDef { + let run_of = |e: PyExpr| call(name("_pf_run"), vec![e]); + let def = |fn_name: &str, params: &[&str], body: Vec, is_async: bool| PyStmt::FuncDef { name: fn_name.to_string(), params: params.iter().map(|p| p.to_string()).collect(), body, - is_async: true, + is_async, + }; + let cold = |args: Vec| PyStmt::Return(call(name("_pf_Async"), args)); + // `def helper(params): async def run(): body; return _pf_Async(run)`. + let cold_helper = |helper: &str, params: &[&str], body: Vec| { + def( + helper, + params, + vec![def("run", &[], body, true), cold(vec![name("run")])], + false, + ) }; + let mut used = used.clone(); + if used + .iter() + .any(|h| h.starts_with("_pf_async_") || h.starts_with("_pf_task_")) + { + used.insert("_pf_Async"); + } + if used.iter().any(|h| { + matches!( + *h, + "_pf_async_parallel" | "_pf_async_race" | "_pf_async_timeout" | "_pf_task_start" + ) + }) { + used.insert("_pf_run"); + } used.iter() .map(|&helper| match helper { - // Async.toThread(f) -> asyncio.to_thread(f, None): the thunk takes the unit value. - "_pf_async_to_thread" => PyStmt::FuncDef { - name: helper.to_string(), - params: vec!["f".to_string()], - body: vec![PyStmt::Return(call( - asyncio("to_thread"), - vec![name("f"), PyExpr::NoneLit], - ))], - is_async: false, - }, - // Async.parallel(xs) -> list(await asyncio.gather(*xs)) - "_pf_async_parallel" => adef( + // class _pf_Async: + // __slots__ = ("fn", "args") + // def __init__(self, fn, *args): self.fn = fn; self.args = args + // def __await__(self): return self.fn(*self.args).__await__() + // async def __call__(self): return await self + "_pf_Async" => { + let this = |a: &str| attr(name("self"), a); + PyStmt::Class { + name: helper.to_string(), + body: vec![ + PyStmt::Assign { + target: "__slots__".to_string(), + value: PyExpr::Tuple(vec![ + PyExpr::Str("fn".to_string()), + PyExpr::Str("args".to_string()), + ]), + }, + def( + "__init__", + &["self", "fn", "*args"], + vec![ + PyStmt::Assign { + target: "self.fn".to_string(), + value: name("fn"), + }, + PyStmt::Assign { + target: "self.args".to_string(), + value: name("args"), + }, + ], + false, + ), + def( + "__await__", + &["self"], + vec![PyStmt::Return(call( + attr( + call(this("fn"), vec![PyExpr::Starred(Box::new(this("args")))]), + "__await__", + ), + vec![], + ))], + false, + ), + // Python code runs a Pyfun value by calling it, as it would + // a zero-argument `async def`: `asyncio.run(job())`. + def( + "__call__", + &["self"], + vec![PyStmt::Return(await_(name("self")))], + true, + ), + def( + "__repr__", + &["self"], + vec![PyStmt::Return(PyExpr::Str("".to_string()))], + false, + ), + ], + } + } + // async def _pf_run(a): return await a + "_pf_run" => def( + helper, + &["a"], + vec![PyStmt::Return(await_(name("a")))], + true, + ), + // Async.sleep(secs) -> _pf_Async(asyncio.sleep, secs) + "_pf_async_sleep" => def( + helper, + &["secs"], + vec![cold(vec![asyncio("sleep"), name("secs")])], + false, + ), + // Async.toThread(f) -> _pf_Async(asyncio.to_thread, f, None): the thunk + // takes the unit value. + "_pf_async_to_thread" => def( + helper, + &["f"], + vec![cold(vec![asyncio("to_thread"), name("f"), PyExpr::NoneLit])], + false, + ), + // Async.parallel(xs): list(await asyncio.gather(*map(_pf_run, xs))) + "_pf_async_parallel" => cold_helper( helper, &["xs"], vec![PyStmt::Return(call( name("list"), vec![await_(call( asyncio("gather"), - vec![PyExpr::Starred(Box::new(name("xs")))], + vec![PyExpr::Starred(Box::new(call( + name("map"), + vec![name("_pf_run"), name("xs")], + )))], ))], ))], ), // Async.race(xs): the first to finish wins and the rest are cancelled. // Racing nothing would wait forever, so an empty list raises a clear // error rather than asyncio's ValueError about an empty set. - "_pf_async_race" => adef( + "_pf_async_race" => cold_helper( helper, &["xs"], vec![ @@ -7070,7 +7406,10 @@ fn async_prelude(used: &BTreeSet<&'static str>, none_singleton: bool) -> Vec, none_singleton: bool) -> Vec Some(await wait_for(c, secs)); None on TimeoutError. - "_pf_async_timeout" => adef( + // Async.timeout(secs, c): Some(await wait_for(_pf_run(c), secs)); None on + // TimeoutError. + "_pf_async_timeout" => cold_helper( helper, &["secs", "c"], vec![PyStmt::Try { @@ -7105,7 +7445,7 @@ fn async_prelude(used: &BTreeSet<&'static str>, none_singleton: bool) -> Vec, none_singleton: bool) -> Vec Ok(await c); Error(_Exception(kind, message)) on Exception. - "_pf_async_catch" => adef( + // Async.catch(c): Ok(await c); Error(_Exception(kind, message)) on Exception. + "_pf_async_catch" => cold_helper( helper, &["c"], vec![PyStmt::Try { @@ -7136,18 +7476,20 @@ fn async_prelude(used: &BTreeSet<&'static str>, none_singleton: bool) -> Vec adef( + // async def scoped(): + // async with asyncio.TaskGroup() as tg: + // _pf_scope.set(tg) + // return await body + // return await asyncio.create_task(scoped()) + // return _pf_Async(run) + "_pf_task_scope" => cold_helper( helper, &["body"], vec![ - adef( - "run", + def( + "scoped", &[], vec![PyStmt::AsyncWith { context: call(asyncio("TaskGroup"), vec![]), @@ -7160,26 +7502,27 @@ fn async_prelude(used: &BTreeSet<&'static str>, none_singleton: bool) -> Vec PyStmt::FuncDef { - name: helper.to_string(), - params: vec!["c".to_string()], - body: vec![ + // Task.start(c): the enclosing scope's TaskGroup owns a fresh run of `c`. + "_pf_task_start" => def( + helper, + &["c"], + vec![ PyStmt::Expr(call( attr(call(attr(name("_pf_scope"), "get"), vec![]), "create_task"), - vec![name("c")], + vec![run_of(name("c"))], )), PyStmt::Return(PyExpr::NoneLit), ], - is_async: false, - }, + false, + ), other => unreachable!("unknown Async helper {other}"), }) .collect() diff --git a/src/lowering/self_tail_call.rs b/src/lowering/self_tail_call.rs index 3e24152..ed7a2b0 100644 --- a/src/lowering/self_tail_call.rs +++ b/src/lowering/self_tail_call.rs @@ -107,14 +107,16 @@ pub(super) fn rewrite(name: &str, params: &[String], body: Vec) -> Outco } /// The async form of the rewrite (`DESIGN.md` §5.4). A `let f a = async { … }` -/// lowers to `def f(a): async def g(): …; return g()`, and its self tail call is -/// `return await f(…)` inside `g` — each message of an agent loop then awaits a -/// fresh coroutine on the same stack, and CPython's ~1000-frame limit is the -/// lifetime cap the loop was written to escape. When the body ends in exactly -/// that wrapper shape, the awaited tail calls in `g` rebind the *outer* -/// parameters (they are `g`'s closure, so a `nonlocal` declaration makes them -/// writable) and go round a `while True:` inside the one coroutine. The -/// preconditions are the sync pass's, applied to `g`'s body. +/// lowers to `def f(a): async def g(): …; return _pf_Async(g)`, and its self tail +/// call is `return await f(…)` inside `g` — each message of an agent loop then +/// awaits a fresh coroutine on the same stack, and CPython's ~1000-frame limit is +/// the lifetime cap the loop was written to escape. When the body ends in exactly +/// that wrapper shape, `g` takes the parameters as its own +/// (`async def g(a): while True: …`, returned as `_pf_Async(g, a)`): the awaited +/// tail calls rebind them and go round the loop inside the one coroutine, and +/// since an `Async` is cold, every await of `f(…)` starts again from the +/// arguments it was built with. The preconditions are the sync pass's, applied +/// to `g`'s body. fn rewrite_async_wrapper( name: &str, params: &[String], @@ -135,8 +137,8 @@ fn rewrite_async_wrapper( PyStmt::Return(PyExpr::Call { func, args }), ) => { g_params.is_empty() - && args.is_empty() - && matches!(func.as_ref(), PyExpr::Name(f) if f == g) + && matches!(func.as_ref(), PyExpr::Name(c) if c == "_pf_Async") + && matches!(args.as_slice(), [PyExpr::Name(f)] if f == g) } _ => false, }; @@ -197,27 +199,22 @@ fn rewrite_async_wrapper( let mut found = false; rewrite_await_stmts(&mut g_rest, name, params, &mut found); debug_assert!(found, "has_await_self_tail_call said there was one"); - // The parameters are the enclosing def's; rebinding them from inside `g` - // needs a `nonlocal` for each one not already declared (a `mut` capture the - // block reassigns may have declared its own). - let declared: HashSet<&String> = g_decls - .iter() - .flat_map(|s| match s { - PyStmt::Global(names) | PyStmt::Nonlocal(names) => names.iter().collect::>(), - _ => Vec::new(), - }) - .collect(); - let fresh: Vec = params - .iter() - .filter(|p| !declared.contains(p)) - .cloned() - .collect(); + // The loop variables are `g`'s own parameters, bound afresh on every await. let mut new_body = g_decls; - if !fresh.is_empty() { - new_body.push(PyStmt::Nonlocal(fresh)); - } new_body.push(PyStmt::WhileTrue { body: g_rest }); *g_body = new_body; + if let PyStmt::FuncDef { + params: g_params, .. + } = &mut rest[n - 2] + { + *g_params = params.to_vec(); + } + rest[n - 1] = PyStmt::Return(PyExpr::Call { + func: Box::new(PyExpr::Name("_pf_Async".to_string())), + args: std::iter::once(PyExpr::Name(g_name)) + .chain(params.iter().map(|p| PyExpr::Name(p.clone()))) + .collect(), + }); Outcome::kept(join(decls, rest)) } diff --git a/src/python_emitter/mod.rs b/src/python_emitter/mod.rs index a82311a..734675b 100644 --- a/src/python_emitter/mod.rs +++ b/src/python_emitter/mod.rs @@ -146,6 +146,9 @@ pub enum PyStmt { body: Vec, is_async: bool, }, + /// `class name: body`: a plain class whose body is ordinary statements. The + /// runtime's cold `_pf_Async` uses it; ADTs and records use [`PyStmt::ClassDef`]. + Class { name: String, body: Vec }, /// `yield value` Yield(PyExpr), /// `yield from value` @@ -524,6 +527,10 @@ fn emit_stmt(stmt: &PyStmt, depth: usize, out: &mut String) { line(out, depth, &format!("{kw} {name}({}):", params.join(", "))); emit_block(body, depth + 1, out); } + PyStmt::Class { name, body } => { + line(out, depth, &format!("class {name}:")); + emit_block(body, depth + 1, out); + } PyStmt::TypedFuncDef { name, params, @@ -914,7 +921,21 @@ fn emit_expr(e: &PyExpr, parent_prec: u8) -> String { emit_expr(lower, 0), emit_expr(upper, 0) ), - PyExpr::Await(inner) => format!("await {}", emit_expr(inner, 100)), + // Awaiting a cold `_pf_Async(f, *args)` the moment it is built is exactly + // `await f(*args)`, which reads as the Python it is (`await inbox.get()`). + PyExpr::Await(inner) => match inner.as_ref() { + PyExpr::Call { func, args } + if matches!(func.as_ref(), PyExpr::Name(n) if n == "_pf_Async") + && !args.is_empty() => + { + let direct = PyExpr::Call { + func: Box::new(args[0].clone()), + args: args[1..].to_vec(), + }; + format!("await {}", emit_expr(&direct, 100)) + } + _ => format!("await {}", emit_expr(inner, 100)), + }, // Emit the operand at `not`'s own level so comparisons stay bare // (`not a == b`) while looser `and`/`or` get parenthesized. PyExpr::Not(inner) => format!("not {}", emit_expr(inner, 4)), diff --git a/src/python_emitter/py311.rs b/src/python_emitter/py311.rs index 26cf639..f98431f 100644 --- a/src/python_emitter/py311.rs +++ b/src/python_emitter/py311.rs @@ -57,7 +57,9 @@ fn rewrite_stmt(stmt: &mut PyStmt) { PyStmt::Return(e) | PyStmt::Expr(e) | PyStmt::Yield(e) | PyStmt::YieldFrom(e) => { rewrite_expr(e) } - PyStmt::FuncDef { body, .. } | PyStmt::TypedFuncDef { body, .. } => { + PyStmt::FuncDef { body, .. } + | PyStmt::TypedFuncDef { body, .. } + | PyStmt::Class { body, .. } => { for s in body { rewrite_stmt(s); } diff --git a/tests/compile.rs b/tests/compile.rs index 31cbb33..76a1582 100644 --- a/tests/compile.rs +++ b/tests/compile.rs @@ -3805,15 +3805,23 @@ fn an_ordinary_function_says_nothing() { #[test] fn an_async_self_tail_call_becomes_a_loop_in_the_coroutine() { - // §5.4's async form: the awaited tail call rebinds the outer parameters - // through a `nonlocal` and loops inside the one coroutine. + // §5.4's async form: the coroutine takes the parameters as its own, and the + // awaited tail call rebinds them and loops inside it. No `nonlocal`: since an + // `Async` is cold, each await starts again from the arguments it was built with. let py = pyfun::compile( "let finished acc = async { return acc }\n\ let loop n acc =\n async {\n return! (if n == 0 then finished acc else loop (n - 1) (acc + n))\n }", ) .unwrap(); - assert!(py.contains("nonlocal n, acc"), "{py}"); - assert!(py.contains("while True:"), "{py}"); + assert!(!py.contains("nonlocal"), "{py}"); + assert!( + py.contains( + "(n, acc): + while True:" + ), + "{py}" + ); + assert!(py.contains(", n, acc)"), "{py}"); assert!(py.contains("n, acc = (n - 1, acc + n)"), "{py}"); assert!(!py.contains("return await loop("), "{py}"); } @@ -4148,11 +4156,11 @@ fn native_spells_out_the_comparisons_of_an_ordered_sum_type() { } #[test] -fn native_leaves_a_parameter_rebound_through_nonlocal_unannotated() { - // An async self tail call rebinds its parameters from the coroutine. mypyc - // copies an annotated `int` parameter into the closure unboxed there - // (python/mypy#22221), so those parameters stay unannotated; a `mut` local - // rebound by a closure is unaffected and keeps the function's annotations. +fn native_annotates_an_async_tail_call_that_needs_no_nonlocal() { + // An async self tail call loops over the coroutine's own parameters, so no + // parameter is rebound through `nonlocal`: the shape mypyc mishandled when + // the parameter was annotated `int` (python/mypy#22221) no longer arises, + // and the function keeps its annotations like any other. let src = "extern import asyncio\n\ extern runAsync: Async a -> a = asyncio.run\n\ let finished count = async { return count }\n\ @@ -4170,8 +4178,8 @@ fn native_leaves_a_parameter_rebound_through_nonlocal_unannotated() { let Some((py, out)) = run_native(src) else { return; }; - assert!(py.contains("def countdown(n, acc):"), "{py}"); - assert!(py.contains("nonlocal n, acc"), "{py}"); + assert!(py.contains("def countdown(n: int, acc: int):"), "{py}"); + assert!(!py.contains("nonlocal n"), "{py}"); assert!(py.contains("def tally(xs: list[int]) -> int:"), "{py}"); assert_eq!(out, ["10", "6"]); } @@ -4207,6 +4215,106 @@ fn e2e_a_start_joins_the_innermost_open_scope() { ); } +#[test] +fn e2e_an_async_value_is_cold_and_runs_again_on_every_await() { + // F#'s `Async` semantics: building runs nothing, and each await (or each + // element of `Async.parallel`, even the same value twice) is a fresh run. + let Some(python) = python_cmd() else { return }; + let src = "extern import asyncio\n\ + extern runAsync: Async a -> a = asyncio.run\n\ + let job = async {\n \ + print \"run\"\n \ + return 1\n\ + }\n\ + print \"built\"\n\ + let twice = async {\n \ + let! a = job\n \ + let! b = job\n \ + let! both = Async.parallel [job, job]\n \ + return a + b + List.sum both\n\ + }\n\ + print (runAsync twice)"; + let py = pyfun::compile(src).unwrap(); + let out = run_python(&python, &py); + assert_eq!( + out.lines().collect::>(), + ["built", "run", "run", "run", "run", "4"], + "{py}" + ); +} + +#[test] +fn e2e_an_async_extern_passed_by_name_or_partially_applied_stays_cold() { + // `List.map pause xs` and `List.map (sleepThen 0.0) xs` build values through + // the extern's adapting lambda, so each can be awaited twice, as a saturated + // call's can. + let Some(python) = python_cmd() else { return }; + let src = "extern import asyncio\n\ + extern runAsync: Async a -> a = asyncio.run\n\ + extern pause: float -> Async unit = asyncio.sleep\n\ + extern sleepThen: float -> a -> Async a = asyncio.sleep\n\ + let pauses = [0.0, 0.0] |> List.map pause\n\ + let values = [1, 2] |> List.map (sleepThen 0.0)\n\ + let main = async {\n \ + for p in pauses:\n \ + do! p\n \ + do! p\n \ + for v in values:\n \ + let! a = v\n \ + let! b = v\n \ + print (a + b)\n \ + return ()\n\ + }\n\ + runAsync main"; + let py = pyfun::compile(src).unwrap(); + let out = run_python(&python, &py); + assert_eq!(out.lines().collect::>(), ["2", "4"], "{py}"); +} + +#[test] +fn e2e_an_async_callback_handed_to_python_runs() { + // `asyncio.start_server` schedules a handler's result only if it is a real + // coroutine; handed a cold `_pf_Async` it would silently never run, and the + // client would read an empty line instead of the reply. + let Some(python) = python_cmd() else { return }; + let src = "extern import asyncio\n\ + extern runAsync: Async a -> a = asyncio.run\n\ + extern type Reader\n\ + extern type Writer\n\ + extern type Server\n\ + extern type Bytes\n\ + extern startServer: (Reader -> Writer ->{io, async} Async unit) -> string -> int -> Async Server = asyncio.start_server\n\ + extern openConnection: string -> int -> Async (Reader, Writer) = asyncio.open_connection\n\ + extern readline: Reader -> Async Bytes = .readline()\n\ + extern write: Writer -> Bytes -> unit = .write\n\ + extern drain: Writer -> Async unit = .drain()\n\ + extern closeWriter: Writer -> unit = .close()\n\ + extern closeServer: Server -> unit = .close()\n\ + extern encode: string -> Bytes = .encode()\n\ + extern decode: Bytes -> string = .decode()\n\ + let shout reader writer = async {\n \ + let! line = readline reader\n \ + write writer (encode (String.upper (decode line)))\n \ + do! drain writer\n \ + closeWriter writer\n\ + }\n\ + let main = async {\n \ + let! server = startServer shout \"127.0.0.1\" 47321\n \ + let! (reader, writer) = openConnection \"127.0.0.1\" 47321\n \ + write writer (encode \"hello\\n\")\n \ + do! drain writer\n \ + let! reply = Async.timeout 5.0 (readline reader)\n \ + print (reply |> Option.map (fun b -> String.strip (decode b)))\n \ + closeWriter writer\n \ + closeServer server\n\ + }\n\ + runAsync main"; + let py = pyfun::compile(src).unwrap(); + assert!(py.contains("lambda *_pf_a: _pf_run(shout(*_pf_a))"), "{py}"); + let out = run_python(&python, &py); + assert_eq!(out.trim(), "Some('HELLO')", "{py}"); +} + #[test] fn the_cookbook_agent_loops_instead_of_recursing() { // The finding that forced the async form: the agent must outlive the stack. @@ -4216,7 +4324,11 @@ fn the_cookbook_agent_loops_instead_of_recursing() { ); let source = std::fs::read_to_string(&path).unwrap(); let py = pyfun::compile(&source).unwrap(); - assert!(py.contains("nonlocal inbox, count"), "{py}"); + assert!(!py.contains("nonlocal"), "{py}"); + assert!( + py.contains("return _pf_Async(_pf_fn1, inbox, count)"), + "{py}" + ); assert!(py.contains("while True:"), "{py}"); assert!(!py.contains("return await agentLoop("), "{py}"); } @@ -4719,7 +4831,8 @@ fn e2e_async_ce_produces_a_coroutine() { let mut program = pyfun::compile("let twice = async { let! x = async { return 21 } return (x + x) }") .unwrap(); - program.push_str("\nimport asyncio\nprint(asyncio.run(twice))\n"); + // Python runs a Pyfun `Async` by calling it, as it would an `async def`. + program.push_str("\nimport asyncio\nprint(asyncio.run(twice()))\n"); assert_eq!(run_python(&python, &program).trim(), "42"); } @@ -4794,7 +4907,7 @@ fn e2e_async_module_members_run_over_asyncio() { runAsync main"; let program = pyfun::compile(src).unwrap(); assert!( - program.contains("return list(await asyncio.gather(*xs))"), + program.contains("return list(await asyncio.gather(*map(_pf_run, xs)))"), "{program}" ); assert!(program.contains("except TimeoutError:"), "{program}"); @@ -8402,7 +8515,7 @@ fn e2e_an_async_bind_shadowing_an_outer_name_is_renamed() { let mut program = pyfun::compile(src).unwrap(); assert!(program.contains("_n = await "), "{program}"); assert!(program.contains("return _n + m"), "{program}"); - program.push_str("\nimport asyncio\nprint(asyncio.run(r))\n"); + program.push_str("\nimport asyncio\nprint(asyncio.run(r()))\n"); assert_eq!(run_python(&python, &program).trim(), "8"); } diff --git a/tests/project.rs b/tests/project.rs index 8ae34c2..0af1c2f 100644 --- a/tests/project.rs +++ b/tests/project.rs @@ -264,6 +264,48 @@ fn e2e_a_task_started_in_one_module_joins_a_scope_opened_in_another() { } } +#[test] +fn e2e_an_imported_async_extern_is_cold_and_its_argument_starts() { + // A dependent module knows only an imported extern's arity, so the exporting + // module's binding adapts it: the result is a cold `_pf_Async` (awaited twice, + // the Python function runs twice) and an `Async` argument reaches Python as a + // real coroutine (`asyncio.run` on Python 3.12 rejects anything else). + let files = compile( + "Main", + &[ + ( + "Main", + "import Clock\n\ + let tick = Clock.pause 0.0\n\ + let twice = async {\n \ + do! tick\n \ + do! tick\n \ + return 2\n\ + }\n\ + print (Clock.runAsync twice)", + ), + ( + "Clock", + "extern runAsync: Async a -> a = asyncio.run\n\ + extern pause: float -> Async unit = asyncio.sleep", + ), + ], + ); + let clock = file(&files, "clock.py"); + assert!( + clock.contains("runAsync = lambda _pf_p0: asyncio.run(_pf_run(_pf_p0))"), + "{clock}" + ); + assert!( + clock.contains("pause = lambda _pf_p0: _pf_Async(asyncio.sleep, _pf_p0)"), + "{clock}" + ); + let dir = Scratch::new("e2e_cold_async_extern"); + if let Some(out) = run_project(&dir, &files, "main.py") { + assert_eq!(out.trim(), "2"); + } +} + #[test] fn an_import_resolves_from_a_vendored_facade_after_the_siblings() { // `pyfun add` vendors a façade's .pyfun files into .pyfun/facades//;