fix: parse multiline fields in CSVArray::toArray

src/CSVArray.php

@@ -11,35 +11,36 @@ class CSVArray
 {
     public const DEFAULT_EMPTY_VALUE = '';
 
-    /**
-     * TODO: fix parsing multiline-content in csv.
-     *
-     * @return array<int, array<string, string>>
-     */
+    private const FIELD_START = 0;
+    private const FIELD_UNENCLOSED = 1;
+    private const FIELD_ENCLOSED = 2;
+    private const FIELD_AFTER_ENCLOSURE = 3;
+
+    /** @return array<int, array<string, string>> */
     public static function toArray(string $csv, string $delimiter = ';', string $enclosure = '"', string $escape = '\\'): array
     {
         /** @var array<int, array<string, string>> $result */
         $result = [];
 
-        $lines = StringUtil::splitLines($csv);
+        $records = self::splitRecords($csv, $delimiter, $enclosure, $escape);
 
         // The first row of the CSV usually contains the column headers
         // We remove it so it does not show up in the result
-        $firstLine = array_shift($lines);
-        if (is_null($firstLine)) {
+        $headerRecord = array_shift($records);
+        if (is_null($headerRecord)) {
             throw new \Exception('Missing column headers.');
         }
 
         /** @var array<int, string> $columnHeaders */
-        $columnHeaders = str_getcsv($firstLine, $delimiter, $enclosure, $escape);
+        $columnHeaders = str_getcsv($headerRecord, $delimiter, $enclosure, $escape);
 
-        foreach ($lines as $index => $line) {
-            if (! StringUtil::hasContent($line)) {
+        foreach ($records as $index => $record) {
+            if (! StringUtil::hasContent($record)) {
                 continue;
             }
 
             /** @var array<int, string> $entries */
-            $entries = str_getcsv($line, $delimiter, $enclosure, $escape);
+            $entries = str_getcsv($record, $delimiter, $enclosure, $escape);
             foreach ($columnHeaders as $columnIndex => $columnName) {
                 $result[$index + 1][$columnName] = $entries[$columnIndex] ?? self::DEFAULT_EMPTY_VALUE;
             }
@@ -48,6 +49,52 @@ public static function toArray(string $csv, string $delimiter = ';', string $enc
         return $result;
     }
 
+    /**
+     * Follows the quoting rules of str_getcsv: an enclosure only opens at the start of a field.
+     *
+     * @return array<int, string>
+     */
+    private static function splitRecords(string $csv, string $delimiter, string $enclosure, string $escape): array
+    {
+        $records = [];
+        $recordStart = 0;
+        $fieldState = self::FIELD_START;
+
+        $length = strlen($csv);
+        for ($position = 0; $position < $length; ++$position) {
+            $character = $csv[$position];
+
+            if ($fieldState === self::FIELD_ENCLOSED) {
+                if ($character === $escape && $escape !== $enclosure) {
+                    ++$position;
+                } elseif ($character === $enclosure) {
+                    if (($csv[$position + 1] ?? '') === $enclosure) {
+                        ++$position;
+                    } else {
+                        $fieldState = self::FIELD_AFTER_ENCLOSURE;
+                    }
+                }
+            } elseif ($character === "\r" || $character === "\n") {
+                $records[] = substr($csv, $recordStart, $position - $recordStart); // @phpstan-ignore-line theCodingMachineSafe.function (safe from PHP 8.0)
+                if ($character === "\r" && ($csv[$position + 1] ?? '') === "\n") {
+                    ++$position;
+                }
+                $recordStart = $position + 1;
+                $fieldState = self::FIELD_START;
+            } elseif ($fieldState === self::FIELD_START && $character === $enclosure) {
+                $fieldState = self::FIELD_ENCLOSED;
+            } elseif ($character === $delimiter) {
+                $fieldState = self::FIELD_START;
+            } elseif ($fieldState === self::FIELD_START && strpos(" \t\v\f", $character) === false) {
+                $fieldState = self::FIELD_UNENCLOSED;
+            }
+        }
+
+        $records[] = substr($csv, $recordStart); // @phpstan-ignore-line theCodingMachineSafe.function (safe from PHP 8.0)
+
+        return $records;
+    }
+
     /** @param iterable<array<string, CSVPrimitive>> $data */
     public static function toCSV(iterable $data, string $delimiter = ';', string $lineSeparator = StringUtil::WINDOWS_NEWLINE): string
     {

tests/CSVArrayTest.php

@@ -163,6 +163,126 @@ public function testToArrayDefaultsMissingColumnsToEmptyStrings(): void
         );
     }
 
+    /** @return iterable<string, array{string, array<int, array<string, string>>}> */
+    public static function lineBreakVariants(): iterable
+    {
+        yield 'carriage return only' => [
+            "a;b\rx;y\r1;2\r",
+            [
+                1 => ['a' => 'x', 'b' => 'y'],
+                2 => ['a' => '1', 'b' => '2'],
+            ],
+        ];
+        yield 'mixed line breaks' => [
+            "a;b\rx;y\n1;2\r\n",
+            [
+                1 => ['a' => 'x', 'b' => 'y'],
+                2 => ['a' => '1', 'b' => '2'],
+            ],
+        ];
+        yield 'carriage return only with line feed in enclosed field' => [
+            "a;b\rx;\"hello\nworld\"\r1;2\r",
+            [
+                1 => ['a' => 'x', 'b' => "hello\nworld"],
+                2 => ['a' => '1', 'b' => '2'],
+            ],
+        ];
+        yield 'vertical tab before enclosure' => [
+            "a;b\n\v\"first\nsecond\";x\n",
+            [
+                1 => ['a' => "first\nsecond", 'b' => 'x'],
+            ],
+        ];
+        yield 'enclosed empty value in single column' => [
+            "name\r\n\"\"\r\nBob\r\n",
+            [
+                1 => ['name' => ''],
+                2 => ['name' => 'Bob'],
+            ],
+        ];
+        yield 'escaped enclosure in multiline field' => [
+            "a;b\r\nx;\"say \\\"hi\r\nthere\"\r\n1;2\r\n",
+            [
+                1 => ['a' => 'x', 'b' => "say \\\"hi\r\nthere"],
+                2 => ['a' => '1', 'b' => '2'],
+            ],
+        ];
+    }
+
+    /** @return iterable<string, array{string, array<int, array<string, string>>}> */
+    public static function enclosuresOutsideEnclosedFields(): iterable
+    {
+        yield 'inch mark in unenclosed field' => [
+            "size;name\r\n12\";Disk\r\n14;Other\r\n",
+            [
+                1 => ['size' => '12"', 'name' => 'Disk'],
+                2 => ['size' => '14', 'name' => 'Other'],
+            ],
+        ];
+        yield 'text after closing enclosure' => [
+            "a;b\r\n\"x\"y\";z\r\n1;2\r\n",
+            [
+                1 => ['a' => 'xy"', 'b' => 'z'],
+                2 => ['a' => '1', 'b' => '2'],
+            ],
+        ];
+        yield 'escaped enclosure in unenclosed field' => [
+            "a;b\r\nx\\\";z\r\n1;2\r\n",
+            [
+                1 => ['a' => 'x\\"', 'b' => 'z'],
+                2 => ['a' => '1', 'b' => '2'],
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider enclosuresOutsideEnclosedFields
+     *
+     * @param array<int, array<string, string>> $array
+     */
+    #[DataProvider('enclosuresOutsideEnclosedFields')]
+    public function testToArrayTreatsEnclosuresOutsideEnclosedFieldsAsText(string $csv, array $array): void
+    {
+        self::assertSame($array, CSVArray::toArray($csv));
+    }
+
+    public function testToArrayWithLineFeedAsEscape(): void
+    {
+        self::assertSame(
+            [
+                1 => ['a' => 'x', 'b' => "hello\n\"world\nend"],
+            ],
+            CSVArray::toArray("a;b\nx;\"hello\n\"world\nend\"\n", ';', '"', "\n")
+        );
+    }
+
+    public function testToArrayWithEscapeEqualToEnclosure(): void
+    {
+        self::assertSame(
+            [
+                1 => ['a' => 'x"y', 'b' => 'z'],
+                2 => ['a' => "m\r\nn", 'b' => '2'],
+            ],
+            CSVArray::toArray("a;b\r\n\"x\"\"y\";z\r\n\"m\r\nn\";2\r\n", ';', '"', '"')
+        );
+    }
+
+    /**
+     * @dataProvider lineBreakVariants
+     *
+     * @param array<int, array<string, string>> $array
+     */
+    #[DataProvider('lineBreakVariants')]
+    public function testToArrayHandlesLineBreakVariants(string $csv, array $array): void
+    {
+        self::assertSame($array, CSVArray::toArray($csv));
+    }
+
+    public function testToArrayYieldsNoRecordsForEmptyCSV(): void
+    {
+        self::assertSame([], CSVArray::toArray(''));
+    }
+
     public function testHandlesMultilineStrings(): void
     {
         $multilineCsv = <<<CSV
@@ -195,8 +315,6 @@ public function testHandlesMultilineStrings(): void
 
         self::assertSame($multilineCsv, CSVArray::toCSV($multilineArray));
 
-        self::markTestIncomplete('This does not work correctly yet');
-        // @phpstan-ignore-next-line https://github.com/phpstan/phpstan-phpunit/issues/52
         self::assertSame($multilineArray, CSVArray::toArray($multilineCsv));
     }
 }