From 5bcd8e6cc2c9e455b1e6e971f9532a7ca5d3c915 Mon Sep 17 00:00:00 2001
From: simontreanor <simon.treanor@gmail.com>
Date: Sun, 18 Oct 2026 00:49:36 +0100
Subject: [PATCH] lexer: a line break in a string is \n whatever the file's
 line endings

A Pyfun file saved with Windows line endings put a carriage return before
every newline of a multi-line string: `"""a<CRLF>b"""` had length 4 where
Python's universal newlines give 3. Students on Windows would carry it into
every triple-quoted string they print, write to a file or compare. It showed
up as the one difference between the Windows and Linux runs of
examples/hello.pyfun in the pre-release Python 3.12 check.

push_char, which takes every literal character of a plain, raw and
interpolated string, now turns `\r\n` and a lone `\r` into `\n`, as Python
does. An escaped `\r` is still a carriage return. Spans are untouched, since
only the literal's content changes.
---
 src/lexer/mod.rs | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index dc04382..aa48722 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -771,7 +771,21 @@ impl<'a> Lexer<'a> {
     /// is valid UTF-8; pushing `b as char` per byte instead would turn every
     /// non-ASCII byte into its own Latin-1 codepoint, and re-encoding on emit would
     /// double-UTF-8-encode the text (`"café"` → mojibake).
+    /// Push one source character of a string literal's content. A line break is
+    /// `\n` whatever the file's line endings, as Python reads source with
+    /// universal newlines: a multi-line string in a file saved on Windows would
+    /// otherwise carry a `\r` before every `\n` (`\r\n` and a lone `\r` both
+    /// become `\n`).
     fn push_char(&mut self, out: &mut String) {
+        if self.src[self.pos] == b'\r' {
+            self.pos += if self.src.get(self.pos + 1) == Some(&b'\n') {
+                2
+            } else {
+                1
+            };
+            out.push('\n');
+            return;
+        }
         let start = self.pos;
         let end = (start + utf8_len(self.src[start])).min(self.src.len());
         match std::str::from_utf8(&self.src[start..end]) {
@@ -1568,6 +1582,34 @@ mod tests {
         assert!(matches!(parts[0], FStrPart::Hole(_)));
     }
 
+    #[test]
+    fn a_windows_line_break_in_a_string_is_a_newline() {
+        // A file saved with `\r\n` line endings gives the same strings as one
+        // with `\n`, as Python's universal newlines do, in every string form; an
+        // escaped `\r` is still a carriage return.
+        assert_eq!(
+            kinds("\"\"\"a\r\nb\"\"\""),
+            vec![Tok::Str("a\nb".to_string()), Tok::Eof]
+        );
+        assert_eq!(
+            kinds("r\"\"\"a\r\nb\"\"\""),
+            vec![Tok::Str("a\nb".to_string()), Tok::Eof]
+        );
+        let toks = kinds("f\"\"\"{x}\r\nb\"\"\"");
+        let Tok::FStr(parts) = &toks[0] else {
+            panic!("expected FStr, got {:?}", toks[0]);
+        };
+        assert_eq!(parts[1], FStrPart::Lit("\nb".to_string()));
+        assert_eq!(
+            kinds("\"\"\"a\rb\"\"\""),
+            vec![Tok::Str("a\nb".to_string()), Tok::Eof]
+        );
+        assert_eq!(
+            kinds("\"a\\rb\""),
+            vec![Tok::Str("a\rb".to_string()), Tok::Eof]
+        );
+    }
+
     #[test]
     fn triple_quoted_raw_string() {
         // `r"""` combines: multi-line + literal backslashes.