Reader recovery: partial trees and every error (ADR 0009 phase 4)

CLAUDE.md

@@ -69,8 +69,8 @@ row as a conformance claim.
 package format · 0004 compiling procedure bodies · 0005 mutable pairs · 0006 errors
 as abnormal passes · 0007 identity for derived combiners · 0008 caching compiled
 bodies (proposed, deliberately not built) · 0009 editor tooling from the runtime
-outward (`ik check --json`, environment enumeration, and the `ik lsp` language
-server are its phases 1–3).
+outward (`ik check --json`, environment enumeration, the `ik lsp` language
+server, and reader recovery are its phases 1–4).
 
 They record measurements and rejected options, not just decisions. When a prediction
 in one turns out wrong, correct it in place rather than leaving it — several carry

IronKernel.Tests/CheckTests.fs

@@ -98,3 +98,10 @@ let ``project check walks sources main and tests`` () =
         Assert.Equal(2, List.length findings)
         Assert.Contains(findings, fun finding -> finding.path = brokenSource)
         Assert.Contains(findings, fun finding -> finding.path = brokenTest))
+
+[<Fact>]
+let ``check reports every broken region in one file`` () =
+    withSource ")\n(ok 1)\n)\n(ok2 2)\n" (fun path ->
+        let findings = checkFile Unrestricted path
+        Assert.Equal(2, List.length findings)
+        Assert.Equal(2, List.length (parseReport (toJson findings))))

IronKernel.Tests/LspTests.fs

@@ -114,12 +114,9 @@ let ``completion merges buffer defines with environment symbols`` () =
     let frames =
         session
             [ request 1 "initialize" "{}"
-              didOpen "(define completion-probe 1)\n"
-              // Mid-edit the buffer no longer parses; defines from the last
-              // good parse must still complete.
-              notification
-                  "textDocument/didChange"
-                  """{"textDocument":{"uri":"file:///probe.ikr"},"contentChanges":[{"text":"(define completion-probe 1)\n(completion-pr"}]}"""
+              // The buffer does not parse mid-edit; the recovering reader
+              // still serves its defines -- no didChange, no cache.
+              didOpen "(define completion-probe 1)\n(completion-pr"
               request 2 "textDocument/completion"
                   """{"textDocument":{"uri":"file:///probe.ikr"},"position":{"line":1,"character":14}}"""
               notification
@@ -209,3 +206,28 @@ let ``unknown requests get a method-not-found error`` () =
             | true, value -> Some value
             | _ -> None)
     Assert.Equal(-32601, error.GetProperty("code").GetInt32())
+
+[<Fact>]
+let ``a broken buffer still gets tokens defines and every error`` () =
+    let frames =
+        session
+            [ request 1 "initialize" "{}"
+              didOpen "(define twice (vau (x) e x))\n)\n(twice"
+              request 2 "textDocument/semanticTokens/full"
+                  """{"textDocument":{"uri":"file:///probe.ikr"}}""" ]
+    // Every broken region publishes its own diagnostic.
+    match diagnosticsOf frames with
+    | [ published ] -> Assert.Equal(2, published.GetArrayLength())
+    | published -> failwithf "expected one publication, got %d" published.Length
+    // The recovered tree still classifies the trailing `(twice` use as an
+    // operative -- the buffer's own vau define survives the breakage.
+    let data =
+        (resultOf 2 frames).GetProperty("data").EnumerateArray()
+        |> Seq.map (fun value -> value.GetInt32())
+        |> List.ofSeq
+    Assert.Equal<int list>(
+        [ 0; 1; 6; 0; 0      // define
+          0; 7; 5; 0; 0      // twice (definition)
+          0; 7; 3; 0; 0      // vau
+          2; 1; 5; 0; 0 ],   // twice (use, on the completed trailing form)
+        data)

IronKernel.Tests/ParserTests.fs

@@ -317,3 +317,90 @@ let ``integer literals are exact at any width`` () =
     Assert.IsType<System.Numerics.BigInteger>(parsed "123456789012345678901234567890") |> ignore
     // The L suffix still forces at least 64-bit.
     Assert.IsType<int64>(parsed "42L") |> ignore
+
+// ---- Recovering reader (ADR 0009 phase 4) ----
+
+let private recoveringErrorLines errors =
+    errors
+    |> List.map (fun error ->
+        match error with
+        | LocatedError(span, _, Parser _) -> span.startPosition.line
+        | other -> failwithf "unexpected error shape: %A" other)
+
+[<Fact>]
+let ``recovering reader matches the strict reader on clean input`` () =
+    let input = "(define a 1)\n(define b (lambda (x) x))\n"
+    let forms, errors = readLocatedExprListRecovering "recover.ikr" input
+    Assert.Empty errors
+    match readLocatedExprList "recover.ikr" input with
+    | Choice2Of2 strict -> Assert.Equal(List.length strict, List.length forms)
+    | Choice1Of2 error -> failwithf "strict reader failed: %A" error
+
+[<Fact>]
+let ``recovery resumes at the next top-level form`` () =
+    // The unclosed first form is lost; the good form after it is not, which
+    // is what the strict reader cannot do.
+    let input = "(broken\n(define after 1)\n"
+    let forms, errors = readLocatedExprListRecovering "recover.ikr" input
+    Assert.Equal(1, List.length errors)
+    match forms with
+    | [ { kind = LList ({ kind = LAtom "define" } :: { kind = LAtom "after" } :: _) } ] -> ()
+    | other -> failwithf "unexpected recovery: %A" other
+
+[<Fact>]
+let ``a stray closer costs one error and keeps both neighbours`` () =
+    let input = "(a 1))\n(b 2)\n"
+    let forms, errors = readLocatedExprListRecovering "recover.ikr" input
+    Assert.Equal(1, List.length errors)
+    Assert.Equal(2, List.length forms)
+
+[<Fact>]
+let ``each broken region reports its own error`` () =
+    let input = ")\n(ok 1)\n)\n(ok2 2)\n"
+    let forms, errors = readLocatedExprListRecovering "recover.ikr" input
+    Assert.Equal<int64 list>([1L; 3L], recoveringErrorLines errors)
+    let formLines = forms |> List.map (fun form -> form.span.startPosition.line)
+    Assert.Equal<int64 list>([2L; 4L], formLines)
+
+[<Fact>]
+let ``the trailing form is completed so mid-edit buffers keep their tree`` () =
+    let input = "(define done 1)\n(define partial (lambda (x)\n"
+    let forms, errors = readLocatedExprListRecovering "recover.ikr" input
+    Assert.Equal(1, List.length errors)
+    match forms with
+    | [ first; second ] ->
+        (match first.kind with
+         | LList ({ kind = LAtom "define" } :: { kind = LAtom "done" } :: _) -> ()
+         | other -> failwithf "unexpected first form: %A" other)
+        (match second.kind with
+         | LList ({ kind = LAtom "define" } :: { kind = LAtom "partial" } :: _) -> ()
+         | other -> failwithf "unexpected completed form: %A" other)
+        Assert.Equal(2L, second.span.startPosition.line)
+    | other -> failwithf "expected two forms, got %A" other
+
+[<Fact>]
+let ``recovered spans survive the window remap exactly`` () =
+    let input = ")\n(marker 1)\n"
+    let forms, _ = readLocatedExprListRecovering "recover.ikr" input
+    match forms with
+    | [ { kind = LList ({ kind = LAtom "marker"; span = atomSpan } :: _); span = formSpan } ] ->
+        Assert.Equal(2L, formSpan.startPosition.line)
+        Assert.Equal(1L, formSpan.startPosition.column)
+        Assert.Equal(2L, atomSpan.startPosition.line)
+        Assert.Equal(2L, atomSpan.startPosition.column)
+        Assert.Equal(6L, atomSpan.endPosition.offset - atomSpan.startPosition.offset)
+    | other -> failwithf "unexpected forms: %A" other
+
+[<Fact>]
+let ``an unclosed string disables trailing completion`` () =
+    let input = "(define ok 1)\n(define s \"unterminated\n"
+    let forms, errors = readLocatedExprListRecovering "recover.ikr" input
+    Assert.Equal(1, List.length errors)
+    Assert.Equal(1, List.length forms)
+
+[<Fact>]
+let ``the nesting bomb stays a single error with no forms`` () =
+    let input = String.replicate 300 "("
+    let forms, errors = readLocatedExprListRecovering "recover.ikr" input
+    Assert.Empty forms
+    Assert.Equal(1, List.length errors)

IronKernel/Check.fs

@@ -20,9 +20,8 @@ module Check =
     }
 
     let checkFile profile (path: string) : Finding list =
-        match Emit.checkSourceFileForProfile profile path with
-        | Choice1Of2 error -> [ { path = path; error = error } ]
-        | Choice2Of2 () -> []
+        Emit.checkFileDiagnostics profile path
+        |> List.map (fun error -> { path = path; error = error })
 
     /// The files the project's author edits: sources, main, tests. Dependency
     /// sources are published artifacts and are not this project's to fix.

IronKernel/Emit.fs

@@ -165,18 +165,22 @@ module Emit =
                 | Choice1Of2 e -> throwError e
                 | Choice2Of2 expressions -> writeIkcPackage outputPath expressions
 
-    /// Everything `compile` checks -- parse, analyze -- on in-memory source.
-    let checkSourceInEnv env sourceName source : ThrowsError<unit> =
-        match analyzePackage env sourceName source with
-        | Choice1Of2 e -> throwError e
-        | Choice2Of2 _ -> returnM ()
-
-    /// Everything `compile` checks -- read, parse, analyze -- with nothing written.
-    let checkSourceFileForProfile profile (inputPath: string) : ThrowsError<unit> =
+    /// Everything `compile` checks, with recovery: every parse error in the
+    /// source (one per broken region), and the same analysis `compile` runs
+    /// over every form that did parse -- analysis cannot fail today, but the
+    /// semantic checks that land there will surface here for free.
+    let checkSourceDiagnostics env sourceName (source: string) : LispError list =
+        let forms, errors = Parser.readLocatedExprListRecovering sourceName source
+        for form in forms do
+            Analyze.analyzeLocatedGuarded env source form |> ignore
+        errors
+
+    /// The same on a file; an unreadable file is itself the one diagnostic.
+    let checkFileDiagnostics profile (inputPath: string) : LispError list =
         match readSource inputPath with
-        | Choice1Of2 e -> throwError e
+        | Choice1Of2 e -> [e]
         | Choice2Of2 source ->
-            checkSourceInEnv (makePrimitiveBindingsForProfile profile) inputPath source
+            checkSourceDiagnostics (makePrimitiveBindingsForProfile profile) inputPath source
 
     [<Obsolete("Use compileFileToPackage; IKC files are packages, not CLR assemblies.")>]
     let compileFileToAssembly inputPath outputPath =

IronKernel/LanguageServer.fs

@@ -181,9 +181,9 @@ module LanguageServer =
         | _ -> acc
 
     let private parsedForms sourceName text =
-        match Parser.readLocatedExprList sourceName text with
-        | Choice2Of2 forms -> forms
-        | Choice1Of2 _ -> []
+        // Recovering: a broken buffer still yields the forms that do parse,
+        // including the trailing form with its brackets closed.
+        Parser.readLocatedExprListRecovering sourceName text |> fst
 
 
     // ---- Contracts and hover text -----------------------------------------
@@ -234,12 +234,6 @@ module LanguageServer =
 
     type private Session = {
         documents : Dictionary<string, string>
-        /// Buffer defines from each document's last *successful* parse. A
-        /// half-typed buffer does not parse (the reader has no error
-        /// recovery, ADR 0009 phase 4), and completion mid-edit is exactly
-        /// when the buffer is broken -- so the last good parse serves until
-        /// the next one.
-        defines : Dictionary<string, Map<string, SymbolClass>>
         /// Bootstrapped kernel environment: symbol source for completion,
         /// hover, and semantic classification. Never evaluated into by the
         /// server -- buffers are parsed and analyzed, not run.
@@ -255,16 +249,11 @@ module LanguageServer =
             if parsed.IsFile then parsed.LocalPath else uri
         with _ -> uri
 
-    let private refreshDefines session (uri: string) text =
-        match Parser.readLocatedExprList (uriToPath uri) text with
-        | Choice2Of2 forms ->
-            session.defines.[uri] <- List.fold collectDefines Map.empty forms
-        | Choice1Of2 _ -> ()
-
-    let private definesFor session (uri: string) =
-        match session.defines.TryGetValue uri with
-        | true, value -> value
-        | _ -> Map.empty
+    /// Defines straight from the current buffer: the recovering reader
+    /// keeps them available mid-edit, which retired the last-good-parse
+    /// cache phase 3 carried (ADR 0009 phase 4).
+    let private definesIn (uri: string) text =
+        parsedForms (uriToPath uri) text |> List.fold collectDefines Map.empty
 
     let private classifySymbol session defines name =
         match Map.tryFind name defines with
@@ -292,9 +281,7 @@ module LanguageServer =
             writer.WriteString("uri", uri)
             writer.WritePropertyName "diagnostics"
             writer.WriteStartArray()
-            match Emit.checkSourceInEnv session.checkEnv sourceName text with
-            | Choice2Of2 () -> ()
-            | Choice1Of2 error ->
+            for error in Emit.checkSourceDiagnostics session.checkEnv sourceName text do
                 let locations, core = errorLocations error
                 writer.WriteStartObject()
                 writer.WritePropertyName "range"
@@ -340,7 +327,7 @@ module LanguageServer =
             | _ -> ""
         let offset = offsetAt text line character
         let prefix, envMatches = Repl.completionCandidates session.env text offset
-        let defines = definesFor session uri
+        let defines = definesIn uri text
         let fromBuffer =
             if prefix = "" then []
             else
@@ -374,7 +361,7 @@ module LanguageServer =
         match symbolAt text offset with
         | None -> respond output id (fun writer -> writer.WriteNullValue())
         | Some name ->
-            let defines = definesFor session uri
+            let defines = definesIn uri text
             let lines =
                 match SymbolTable.getVar' session.env name with
                 | Some value ->
@@ -440,7 +427,7 @@ module LanguageServer =
             | true, value -> value
             | _ -> ""
         let sourceName = uriToPath uri
-        let data = semanticTokenData (classifySymbol session (definesFor session uri)) sourceName text
+        let data = semanticTokenData (classifySymbol session (definesIn uri text)) sourceName text
         respond output id (fun writer ->
             writer.WriteStartObject()
             writer.WritePropertyName "data"
@@ -468,7 +455,6 @@ module LanguageServer =
         | Choice2Of2 env ->
             let session = {
                 documents = Dictionary()
-                defines = Dictionary()
                 env = env
                 checkEnv = Runtime.makePrimitiveBindingsForProfile profile
             }
@@ -530,7 +516,6 @@ module LanguageServer =
                         let textDocument = parameters.GetProperty "textDocument"
                         let uri = textDocument.GetProperty("uri").GetString()
                         session.documents.[uri] <- textDocument.GetProperty("text").GetString()
-                        refreshDefines session uri session.documents.[uri]
                         publishDiagnostics output session uri
                     | "textDocument/didChange" ->
                         let uri = textDocumentUri parameters
@@ -542,13 +527,11 @@ module LanguageServer =
                         match text with
                         | Some value ->
                             session.documents.[uri] <- value
-                            refreshDefines session uri value
                             publishDiagnostics output session uri
                         | None -> ()
                     | "textDocument/didClose" ->
                         let uri = textDocumentUri parameters
                         session.documents.Remove uri |> ignore
-                        session.defines.Remove uri |> ignore
                         notify output "textDocument/publishDiagnostics" (fun writer ->
                             writer.WriteStartObject()
                             writer.WriteString("uri", uri)