diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..896fc637 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# git should not translate line endings on checkout. +*.csv -text +*.tsv -text diff --git a/dataframe-fastcsv/tests/Operations/Projection.hs b/dataframe-fastcsv/tests/Operations/Projection.hs index 935e94dc..4dbed2c5 100644 --- a/dataframe-fastcsv/tests/Operations/Projection.hs +++ b/dataframe-fastcsv/tests/Operations/Projection.hs @@ -10,7 +10,7 @@ module Operations.Projection (tests) where import qualified Data.Map as M import qualified Data.Text as T -import qualified Data.Text.IO as TIO +import qualified Data.Text.IO.Utf8 as TIO import Control.Exception (SomeException, evaluate, try) import Data.List (isInfixOf) diff --git a/dataframe-fastcsv/tests/Operations/ReadCsv.hs b/dataframe-fastcsv/tests/Operations/ReadCsv.hs index e21b7fe0..1d49148a 100644 --- a/dataframe-fastcsv/tests/Operations/ReadCsv.hs +++ b/dataframe-fastcsv/tests/Operations/ReadCsv.hs @@ -37,7 +37,14 @@ import DataFrame.Internal.DataFrame ( ) import DataFrame.Schema (Schema (..), SchemaType (..)) import System.Directory (removeFile) -import System.IO (IOMode (..), withFile) +import System.IO ( + IOMode (..), + hSetEncoding, + hSetNewlineMode, + noNewlineTranslation, + utf8, + withFile, + ) import Test.HUnit import Type.Reflection (typeRep) @@ -59,6 +66,8 @@ prettyPrintTsv = prettyPrintSeparated '\t' prettyPrintSeparated :: Char -> FilePath -> DataFrame -> IO () prettyPrintSeparated sep filepath df = withFile filepath WriteMode $ \handle -> do + hSetEncoding handle utf8 + hSetNewlineMode handle noNewlineTranslation let (rows, _) = dataframeDimensions df let headers = map fst (L.sortBy (compare `on` snd) (M.toList (columnIndices df))) TIO.hPutStrLn diff --git a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs index c99bd534..98204261 100644 --- a/dataframe-fastcsv/tests/Operations/TypedExtraction.hs +++ b/dataframe-fastcsv/tests/Operations/TypedExtraction.hs @@ -12,7 +12,11 @@ import qualified Data.Map as M import qualified Data.Proxy as P import qualified Data.Text as T import qualified Data.Text.Encoding as TE -import qualified Data.Text.IO as TIO + +-- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the +-- handle's text mode, which on Windows turns \n into \r\n and +-- corrupts inputs meant for byte-level parsers. +import qualified Data.Text.IO.Utf8 as TIO import Control.Exception (ErrorCall, evaluate, try) import Data.Time (Day) diff --git a/dataframe-fastcsv/tests/Properties/Csv.hs b/dataframe-fastcsv/tests/Properties/Csv.hs index f230ca84..81ae5d30 100644 --- a/dataframe-fastcsv/tests/Properties/Csv.hs +++ b/dataframe-fastcsv/tests/Properties/Csv.hs @@ -20,7 +20,7 @@ import qualified Data.List as L import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Text.Encoding as TE -import qualified Data.Text.IO as TIO +import qualified Data.Text.IO.Utf8 as TIO import qualified Data.Vector as V import DataFrame.IO.CSV (defaultReadOptions) @@ -130,7 +130,7 @@ the temp file afterwards no matter what. -} withCsvFile :: String -> T.Text -> (FilePath -> IO a) -> IO a withCsvFile label body action = do - let path = "/tmp/fastcsv_prop_" <> label <> ".csv" + let path = "./tests/data/unstable_csv/fastcsv_prop_" <> label <> ".csv" TIO.writeFile path body r <- action path removeFile path @@ -209,7 +209,7 @@ prop_unclosed_quote_throws = forAll (listOf1 arbitrary) $ \(cells :: [Cell]) -> T.intercalate "," (map (encodeCell ',' . unCell) cells) csv = "v\n" <> plainRow <> ",\"dangling\n" result <- run $ do - let path = "/tmp/fastcsv_prop_unclosed.csv" + let path = "./tests/data/unstable_csv/fastcsv_prop_unclosed.csv" TIO.writeFile path csv r <- try @CsvParseError (D.fastReadCsv path) removeFile path diff --git a/tests/Operations/Record.hs b/tests/Operations/Record.hs index 520eeeab..96321f79 100644 --- a/tests/Operations/Record.hs +++ b/tests/Operations/Record.hs @@ -17,7 +17,7 @@ module Operations.Record where import Data.Int (Int64) import qualified Data.Map.Strict as M import qualified Data.Text as T -import qualified Data.Text.IO as TIO +import qualified Data.Text.IO.Utf8 as TIO import GHC.Generics (Generic) import qualified DataFrame as D @@ -27,6 +27,7 @@ import DataFrame.Operators import qualified DataFrame.Schema as IS import DataFrame.Typed (Schema) import qualified DataFrame.Typed as DT +import System.Directory (getTemporaryDirectory) import Test.HUnit @@ -289,7 +290,8 @@ deriveSchemaReadsCsv = TestCase $ do , "2,eu,20.5" , "3,ap,30.0" ] - tmp = "/tmp/dataframe_test_deriveSchema.csv" + tmpDir <- getTemporaryDirectory + let tmp = tmpDir <> "/dataframe_test_deriveSchema.csv" TIO.writeFile tmp csv df <- D.readCsvWithSchema orderSchema tmp assertEqual diff --git a/tests/Operations/WriteCsv.hs b/tests/Operations/WriteCsv.hs index 9c2364be..b039753c 100644 --- a/tests/Operations/WriteCsv.hs +++ b/tests/Operations/WriteCsv.hs @@ -4,10 +4,11 @@ module Operations.WriteCsv where import qualified Data.Text as T -import qualified Data.Text.IO as TIO +import qualified Data.Text.IO.Utf8 as TIO import qualified DataFrame as D import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame (..), toCsv, toSeparated) +import System.Directory (getTemporaryDirectory) import Test.HUnit -- Basic test: Int and Text columns produce correct CSV @@ -81,7 +82,8 @@ toCsvRoundTrip = TestLabel "toCsv_roundTrip" $ TestCase $ do , ("b", DI.fromList @T.Text ["hello", "world", "test"]) ] let csvText = toCsv df - let tmpPath = "/tmp/dataframe_test_toCsv_roundtrip.csv" + tmpDir <- getTemporaryDirectory + let tmpPath = tmpDir <> "/dataframe_test_toCsv_roundtrip.csv" TIO.writeFile tmpPath csvText df' <- D.readCsv tmpPath assertEqual