From 77c0caa8d07ae85962acad6b4e8cef6b4d94a264 Mon Sep 17 00:00:00 2001 From: skymanbp <57272723+skymanbp@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:32:29 -0400 Subject: [PATCH 01/10] Fix Windows test portability: line-ending translation and /tmp assumptions Three root causes, all test-infrastructure (no library code changed), found by the Windows CI lanes and reproduced on a Windows 11 machine: 1. Missing .gitattributes: with git's core.autocrlf=true (the Windows default), checkout rewrites the CSV fixtures' LF to CRLF - including newlines EMBEDDED IN QUOTED FIELDS (git does not know CSV) - so the byte-level fastcsv reader sees corrupted input and the fast_roundtrip_newlines / fast_roundtrip_quotes_and_newlines round-trips fail. Fixed by marking *.csv / *.tsv -text. 2. Text-mode writes in test harnesses: Data.Text.IO writeFile/hPutStrLn honour the handle's text mode, which on Windows (a) translates \n to \r\n, corrupting quoted embedded newlines (typed_quote_spans_boundary, and the round-trip rewrite path in prettyPrintSeparated), and (b) encodes via the OS locale codepage, crashing fast_roundtrip_utf8 with 'cannot encode character' on non-UTF-8 codepages (e.g. CP936). Fixed by Data.Text.IO.Utf8 (same signatures, byte-mode UTF-8, no new dependency; text >= 2.1 is already required) and, for the streaming pretty-printer, hSetEncoding utf8 + hSetNewlineMode noNewlineTranslation. 3. Hardcoded /tmp paths: toCsv_roundTrip wrote to /tmp (does not exist on Windows); fixed with getTemporaryDirectory. The same pattern in Properties/Csv.hs (two sites) now uses the suite's existing tests/data/unstable_csv scratch dir. Verified on Windows 11 (GHC 9.12.4): dataframe suite 1097/1097, dataframe-fastcsv suite 61/61, zero errors/failures - previously toCsv_roundTrip, fast_roundtrip_newlines, fast_roundtrip_quotes_and_newlines and typed_quote_spans_boundary failed (https://github.com/skymanbp/dataframe/actions/runs/32095552668). Co-Authored-By: Claude Fable 5 --- .gitattributes | 6 ++++++ dataframe-fastcsv/tests/Operations/Projection.hs | 5 ++++- dataframe-fastcsv/tests/Operations/ReadCsv.hs | 8 +++++++- dataframe-fastcsv/tests/Operations/TypedExtraction.hs | 5 ++++- dataframe-fastcsv/tests/Properties/Csv.hs | 9 ++++++--- tests/Operations/WriteCsv.hs | 9 +++++++-- 6 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..235c4640 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# CSV/TSV test fixtures are byte-exact inputs for byte-level parsers +# (dataframe-fastcsv reads raw bytes); git must never translate their +# line endings on checkout, or quoted embedded newlines gain \r on +# Windows and round-trip tests fail. +*.csv -text +*.tsv -text diff --git a/dataframe-fastcsv/tests/Operations/Projection.hs b/dataframe-fastcsv/tests/Operations/Projection.hs index 935e94dc..46f4a286 100644 --- a/dataframe-fastcsv/tests/Operations/Projection.hs +++ b/dataframe-fastcsv/tests/Operations/Projection.hs @@ -10,7 +10,10 @@ module Operations.Projection (tests) where import qualified Data.Map as M import qualified Data.Text as T -import qualified Data.Text.IO as TIO +-- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the +-- handle's text mode, which on Windows turns \n into \r\n and +-- corrupts inputs meant for byte-level parsers. +import qualified Data.Text.IO.Utf8 as TIO import Control.Exception (SomeException, evaluate, try) import Data.List (isInfixOf) diff --git a/dataframe-fastcsv/tests/Operations/ReadCsv.hs b/dataframe-fastcsv/tests/Operations/ReadCsv.hs index e21b7fe0..8ba5e49d 100644 --- a/dataframe-fastcsv/tests/Operations/ReadCsv.hs +++ b/dataframe-fastcsv/tests/Operations/ReadCsv.hs @@ -37,7 +37,7 @@ import DataFrame.Internal.DataFrame ( ) import DataFrame.Schema (Schema (..), SchemaType (..)) import System.Directory (removeFile) -import System.IO (IOMode (..), withFile) +import System.IO (IOMode (..), hSetEncoding, hSetNewlineMode, noNewlineTranslation, utf8, withFile) import Test.HUnit import Type.Reflection (typeRep) @@ -59,6 +59,12 @@ prettyPrintTsv = prettyPrintSeparated '\t' prettyPrintSeparated :: Char -> FilePath -> DataFrame -> IO () prettyPrintSeparated sep filepath df = withFile filepath WriteMode $ \handle -> do + -- Byte-exact UTF-8 output: a default handle uses the OS locale + -- encoding (crashes on non-ANSI text under a non-UTF-8 codepage) + -- and translates \n to \r\n on Windows, corrupting quoted embedded + -- newlines for the byte-level reader. + hSetEncoding handle utf8 + hSetNewlineMode handle noNewlineTranslation let (rows, _) = dataframeDimensions df let headers = map fst (L.sortBy (compare `on` snd) (M.toList (columnIndices df))) TIO.hPutStrLn diff --git a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs index c99bd534..d22dc505 100644 --- a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs +++ b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs @@ -12,7 +12,10 @@ import qualified Data.Map as M import qualified Data.Proxy as P import qualified Data.Text as T import qualified Data.Text.Encoding as TE -import qualified Data.Text.IO as TIO +-- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the +-- handle's text mode, which on Windows turns \n into \r\n and +-- corrupts inputs meant for byte-level parsers. +import qualified Data.Text.IO.Utf8 as TIO import Control.Exception (ErrorCall, evaluate, try) import Data.Time (Day) diff --git a/dataframe-fastcsv/tests/Properties/Csv.hs b/dataframe-fastcsv/tests/Properties/Csv.hs index f230ca84..e79064ab 100644 --- a/dataframe-fastcsv/tests/Properties/Csv.hs +++ b/dataframe-fastcsv/tests/Properties/Csv.hs @@ -20,7 +20,10 @@ import qualified Data.List as L import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.Encoding as TE -import qualified Data.Text.IO as TIO +-- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the +-- handle's text mode, which on Windows turns \n into \r\n and +-- corrupts inputs meant for byte-level parsers. +import qualified Data.Text.IO.Utf8 as TIO import qualified Data.Vector as V import DataFrame.IO.CSV (defaultReadOptions) @@ -130,7 +133,7 @@ the temp file afterwards no matter what. -} withCsvFile :: String -> T.Text -> (FilePath -> IO a) -> IO a withCsvFile label body action = do - let path = "/tmp/fastcsv_prop_" <> label <> ".csv" + let path = "./tests/data/unstable_csv/fastcsv_prop_" <> label <> ".csv" TIO.writeFile path body r <- action path removeFile path @@ -209,7 +212,7 @@ prop_unclosed_quote_throws = forAll (listOf1 arbitrary) $ \(cells :: [Cell]) -> T.intercalate "," (map (encodeCell ',' . unCell) cells) csv = "v\n" <> plainRow <> ",\"dangling\n" result <- run $ do - let path = "/tmp/fastcsv_prop_unclosed.csv" + let path = "./tests/data/unstable_csv/fastcsv_prop_unclosed.csv" TIO.writeFile path csv r <- try @CsvParseError (D.fastReadCsv path) removeFile path diff --git a/tests/Operations/WriteCsv.hs b/tests/Operations/WriteCsv.hs index 9c2364be..f5e42a31 100644 --- a/tests/Operations/WriteCsv.hs +++ b/tests/Operations/WriteCsv.hs @@ -4,10 +4,14 @@ module Operations.WriteCsv where import qualified Data.Text as T -import qualified Data.Text.IO as TIO +-- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the +-- handle's text mode, which on Windows turns \n into \r\n and +-- corrupts inputs meant for byte-level parsers. +import qualified Data.Text.IO.Utf8 as TIO import qualified DataFrame as D import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame (..), toCsv, toSeparated) +import System.Directory (getTemporaryDirectory) import Test.HUnit -- Basic test: Int and Text columns produce correct CSV @@ -81,7 +85,8 @@ toCsvRoundTrip = TestLabel "toCsv_roundTrip" $ TestCase $ do , ("b", DI.fromList @T.Text ["hello", "world", "test"]) ] let csvText = toCsv df - let tmpPath = "/tmp/dataframe_test_toCsv_roundtrip.csv" + tmpDir <- getTemporaryDirectory + let tmpPath = tmpDir <> "/dataframe_test_toCsv_roundtrip.csv" TIO.writeFile tmpPath csvText df' <- D.readCsv tmpPath assertEqual From 41f2c567da7c58a334abb1fafdb338165386d439 Mon Sep 17 00:00:00 2001 From: skymanbp <57272723+skymanbp@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:35:03 -0400 Subject: [PATCH 02/10] style: fourmolu formatting for the portability fix Co-Authored-By: Claude Fable 5 --- dataframe-fastcsv/tests/Operations/Projection.hs | 1 + dataframe-fastcsv/tests/Operations/ReadCsv.hs | 9 ++++++++- dataframe-fastcsv/tests/Operations/TypedExtraction.hs | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dataframe-fastcsv/tests/Operations/Projection.hs b/dataframe-fastcsv/tests/Operations/Projection.hs index 46f4a286..37a453f3 100644 --- a/dataframe-fastcsv/tests/Operations/Projection.hs +++ b/dataframe-fastcsv/tests/Operations/Projection.hs @@ -10,6 +10,7 @@ module Operations.Projection (tests) where import qualified Data.Map as M import qualified Data.Text as T + -- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the -- handle's text mode, which on Windows turns \n into \r\n and -- corrupts inputs meant for byte-level parsers. diff --git a/dataframe-fastcsv/tests/Operations/ReadCsv.hs b/dataframe-fastcsv/tests/Operations/ReadCsv.hs index 8ba5e49d..b9aba909 100644 --- a/dataframe-fastcsv/tests/Operations/ReadCsv.hs +++ b/dataframe-fastcsv/tests/Operations/ReadCsv.hs @@ -37,7 +37,14 @@ import DataFrame.Internal.DataFrame ( ) import DataFrame.Schema (Schema (..), SchemaType (..)) import System.Directory (removeFile) -import System.IO (IOMode (..), hSetEncoding, hSetNewlineMode, noNewlineTranslation, utf8, withFile) +import System.IO ( + IOMode (..), + hSetEncoding, + hSetNewlineMode, + noNewlineTranslation, + utf8, + withFile, + ) import Test.HUnit import Type.Reflection (typeRep) diff --git a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs index d22dc505..98204261 100644 --- a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs +++ b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs @@ -12,6 +12,7 @@ import qualified Data.Map as M import qualified Data.Proxy as P import qualified Data.Text as T import qualified Data.Text.Encoding as TE + -- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the -- handle's text mode, which on Windows turns \n into \r\n and -- corrupts inputs meant for byte-level parsers. From 68272bc9f90c4c571b9031da5e2292bcae69695e Mon Sep 17 00:00:00 2001 From: skymanbp <57272723+skymanbp@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:57:21 -0400 Subject: [PATCH 03/10] Fix the third /tmp site (deriveSchemaReadsCsv) + remaining format nits A repo-wide sweep for '/tmp' found one more test writing there: deriveSchemaReadsCsv in tests/Operations/Record.hs. It passed on a dev machine that happened to have C:\tmp (Windows resolves /tmp against the drive root) but fails on CI runners. Same fix as WriteCsv.hs: getTemporaryDirectory + the byte-mode UTF-8 writer. The two other grep hits are not test-filesystem uses (a benchmark default argument and a pure URI-predicate check) and are left untouched. Local re-run: dataframe 1097/1097, dataframe-fastcsv 61/61, zero errors/failures. Co-Authored-By: Claude Fable 5 --- dataframe-fastcsv/tests/Properties/Csv.hs | 1 + tests/Operations/Record.hs | 10 ++++++++-- tests/Operations/WriteCsv.hs | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dataframe-fastcsv/tests/Properties/Csv.hs b/dataframe-fastcsv/tests/Properties/Csv.hs index e79064ab..f530267f 100644 --- a/dataframe-fastcsv/tests/Properties/Csv.hs +++ b/dataframe-fastcsv/tests/Properties/Csv.hs @@ -20,6 +20,7 @@ import qualified Data.List as L import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.Encoding as TE + -- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the -- handle's text mode, which on Windows turns \n into \r\n and -- corrupts inputs meant for byte-level parsers. diff --git a/tests/Operations/Record.hs b/tests/Operations/Record.hs index 520eeeab..1df04404 100644 --- a/tests/Operations/Record.hs +++ b/tests/Operations/Record.hs @@ -17,7 +17,11 @@ module Operations.Record where import Data.Int (Int64) import qualified Data.Map.Strict as M import qualified Data.Text as T -import qualified Data.Text.IO as TIO + +-- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the +-- handle's text mode, which on Windows turns \n into \r\n and +-- corrupts inputs meant for byte-level parsers. +import qualified Data.Text.IO.Utf8 as TIO import GHC.Generics (Generic) import qualified DataFrame as D @@ -25,6 +29,7 @@ import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI import DataFrame.Operators import qualified DataFrame.Schema as IS +import System.Directory (getTemporaryDirectory) import DataFrame.Typed (Schema) import qualified DataFrame.Typed as DT @@ -289,7 +294,8 @@ deriveSchemaReadsCsv = TestCase $ do , "2,eu,20.5" , "3,ap,30.0" ] - tmp = "/tmp/dataframe_test_deriveSchema.csv" + tmpDir <- getTemporaryDirectory + let tmp = tmpDir <> "/dataframe_test_deriveSchema.csv" TIO.writeFile tmp csv df <- D.readCsvWithSchema orderSchema tmp assertEqual diff --git a/tests/Operations/WriteCsv.hs b/tests/Operations/WriteCsv.hs index f5e42a31..00826722 100644 --- a/tests/Operations/WriteCsv.hs +++ b/tests/Operations/WriteCsv.hs @@ -4,6 +4,7 @@ module Operations.WriteCsv where import qualified Data.Text as T + -- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the -- handle's text mode, which on Windows turns \n into \r\n and -- corrupts inputs meant for byte-level parsers. From ed1fcbdd04a84d2d5641596dbf462365437022ce Mon Sep 17 00:00:00 2001 From: skymanbp <57272723+skymanbp@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:24:17 -0400 Subject: [PATCH 04/10] Move System.Directory import to fourmolu's sorted position Co-Authored-By: Claude Fable 5 --- tests/Operations/Record.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Operations/Record.hs b/tests/Operations/Record.hs index 1df04404..ff5d8a63 100644 --- a/tests/Operations/Record.hs +++ b/tests/Operations/Record.hs @@ -29,9 +29,9 @@ import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI import DataFrame.Operators import qualified DataFrame.Schema as IS -import System.Directory (getTemporaryDirectory) import DataFrame.Typed (Schema) import qualified DataFrame.Typed as DT +import System.Directory (getTemporaryDirectory) import Test.HUnit From 2530fe9624413fc678642af859d0e753dbba50ff Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Tue, 18 Aug 2026 18:58:44 -0700 Subject: [PATCH 05/10] Remove comments from import site. We can leave them only where they are used. --- dataframe-fastcsv/tests/Operations/Projection.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dataframe-fastcsv/tests/Operations/Projection.hs b/dataframe-fastcsv/tests/Operations/Projection.hs index 37a453f3..4dbed2c5 100644 --- a/dataframe-fastcsv/tests/Operations/Projection.hs +++ b/dataframe-fastcsv/tests/Operations/Projection.hs @@ -10,10 +10,6 @@ module Operations.Projection (tests) where import qualified Data.Map as M import qualified Data.Text as T - --- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the --- handle's text mode, which on Windows turns \n into \r\n and --- corrupts inputs meant for byte-level parsers. import qualified Data.Text.IO.Utf8 as TIO import Control.Exception (SomeException, evaluate, try) From 00e577235267cde9261820431b25a7f96e37fa72 Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Tue, 18 Aug 2026 19:00:38 -0700 Subject: [PATCH 06/10] Code already explains itself We can add explanatory comments in the tests. --- dataframe-fastcsv/tests/Operations/ReadCsv.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dataframe-fastcsv/tests/Operations/ReadCsv.hs b/dataframe-fastcsv/tests/Operations/ReadCsv.hs index b9aba909..1d49148a 100644 --- a/dataframe-fastcsv/tests/Operations/ReadCsv.hs +++ b/dataframe-fastcsv/tests/Operations/ReadCsv.hs @@ -66,10 +66,6 @@ prettyPrintTsv = prettyPrintSeparated '\t' prettyPrintSeparated :: Char -> FilePath -> DataFrame -> IO () prettyPrintSeparated sep filepath df = withFile filepath WriteMode $ \handle -> do - -- Byte-exact UTF-8 output: a default handle uses the OS locale - -- encoding (crashes on non-ANSI text under a non-UTF-8 codepage) - -- and translates \n to \r\n on Windows, corrupting quoted embedded - -- newlines for the byte-level reader. hSetEncoding handle utf8 hSetNewlineMode handle noNewlineTranslation let (rows, _) = dataframeDimensions df From 8a70e140c8a5231c70f628e6261ea00cae124425 Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Tue, 18 Aug 2026 19:01:59 -0700 Subject: [PATCH 07/10] Remove comments from import --- dataframe-fastcsv/tests/Properties/Csv.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dataframe-fastcsv/tests/Properties/Csv.hs b/dataframe-fastcsv/tests/Properties/Csv.hs index f530267f..81ae5d30 100644 --- a/dataframe-fastcsv/tests/Properties/Csv.hs +++ b/dataframe-fastcsv/tests/Properties/Csv.hs @@ -20,10 +20,6 @@ import qualified Data.List as L import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.Encoding as TE - --- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the --- handle's text mode, which on Windows turns \n into \r\n and --- corrupts inputs meant for byte-level parsers. import qualified Data.Text.IO.Utf8 as TIO import qualified Data.Vector as V From 378a4f07fd77a2391d1f2015f068ed13ba8fef09 Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Tue, 18 Aug 2026 19:02:43 -0700 Subject: [PATCH 08/10] Remove comment from import Removed unnecessary blank lines and comments in Record.hs --- tests/Operations/Record.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/Operations/Record.hs b/tests/Operations/Record.hs index ff5d8a63..96321f79 100644 --- a/tests/Operations/Record.hs +++ b/tests/Operations/Record.hs @@ -17,10 +17,6 @@ module Operations.Record where import Data.Int (Int64) import qualified Data.Map.Strict as M import qualified Data.Text as T - --- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the --- handle's text mode, which on Windows turns \n into \r\n and --- corrupts inputs meant for byte-level parsers. import qualified Data.Text.IO.Utf8 as TIO import GHC.Generics (Generic) From fec7000f74020010db45bde9ca3d79a21d53b34f Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Tue, 18 Aug 2026 19:03:08 -0700 Subject: [PATCH 09/10] Remove comments from import --- tests/Operations/WriteCsv.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/Operations/WriteCsv.hs b/tests/Operations/WriteCsv.hs index 00826722..b039753c 100644 --- a/tests/Operations/WriteCsv.hs +++ b/tests/Operations/WriteCsv.hs @@ -4,10 +4,6 @@ module Operations.WriteCsv where import qualified Data.Text as T - --- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the --- handle's text mode, which on Windows turns \n into \r\n and --- corrupts inputs meant for byte-level parsers. import qualified Data.Text.IO.Utf8 as TIO import qualified DataFrame as D import qualified DataFrame.Internal.Column as DI From 6728beee0b5a3d867c8efc08c558fb028dd554a1 Mon Sep 17 00:00:00 2001 From: Michael Chavinda Date: Tue, 18 Aug 2026 19:04:34 -0700 Subject: [PATCH 10/10] Fix gitattribute comment The comment read too much like AI --- .gitattributes | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitattributes b/.gitattributes index 235c4640..896fc637 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,3 @@ -# CSV/TSV test fixtures are byte-exact inputs for byte-level parsers -# (dataframe-fastcsv reads raw bytes); git must never translate their -# line endings on checkout, or quoted embedded newlines gain \r on -# Windows and round-trip tests fail. +# git should not translate line endings on checkout. *.csv -text *.tsv -text