diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index b231bdac5..e9f8cde71 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -849,7 +849,8 @@ TEST(CliE2E, WriteImportsQuotedFieldWithEmbeddedNewline) { std::string csv_path = tsfile_cli_test::unique_temp_path("tsfile_cli_nl_in", ".csv"); { - std::ofstream o(csv_path.c_str()); + // Keep the fixture's LF bytes unchanged on Windows as well. + std::ofstream o(csv_path.c_str(), std::ios::binary); // The note field on the first row spans two physical lines inside // quotes; it must import as a single row, not be split into two. o << "time,id1,note\n0,dev,\"line one\nline two\"\n1,dev,plain\n"; @@ -1985,3 +1986,90 @@ TEST(CliE2E, WriteCatCsvWritePreservesLeadingBackslashes) { std::remove(exported.c_str()); std::remove(second.c_str()); } + +TEST(CliE2E, WriteCatAndExportCsvPreserveQuotedLineEndings) { + const std::string input = + tsfile_cli_test::unique_temp_path("tsfile_cli_crlf_input", ".csv"); + const std::string first = + tsfile_cli_test::unique_temp_path("tsfile_cli_crlf_first", ".tsfile"); + const std::string expected_csv = + "time,site,message,note\n" + "0,\"s\r\n1\",\"first\r\nsecond\",\"\"\"quoted\"\"\r\n\r\nlast\"\n" + "1,\"s\r\n1\",\"lf\nonly\",\"mixed\r\nand\nmore\"\n" + "2,\"s\r\n1\",\"carriage\rreturn\",\"\r\n\"\n" + "3,\"s\r\n1\",\"ends\r\",\"line\r\nend\"\n"; + { + std::ofstream csv(input.c_str(), std::ios::binary); + ASSERT_TRUE(csv.good()); + // CRLF record boundaries, embedded CRLF/LF/CR, and no final newline. + csv << "time,site,message,note\r\n" + "0,\"s\r\n1\",\"first\r\nsecond\"," + "\"\"\"quoted\"\"\r\n\r\nlast\"\r\n" + "1,\"s\r\n1\",\"lf\nonly\",\"mixed\r\nand\nmore\"\r\n" + "2,\"s\r\n1\",\"carriage\rreturn\",\"\r\n\"\r\n" + "3,\"s\r\n1\",\"ends\r\",\"line\r\nend\""; + } + const auto write = [&](const std::string& source, + const std::string& target) { + std::ostringstream out, err; + const int code = tsfile_cli::run_cli( + {"write", "--table", "t", "--tag", "site", "STRING", "--field", + "message", "STRING", "--field", "note", "TEXT", "-i", source, "-o", + target}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + return code; + }; + const auto read = [&](const std::string& path, const std::string& format) { + std::ostringstream out, err; + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", format, path}, + out, err), + 0) + << err.str(); + return out.str(); + }; + const std::string expected_ndjson = + R"json({"time":"0","site":"s\r\n1","message":"first\r\nsecond","note":"\"quoted\"\r\n\r\nlast"})json" + "\n" + R"json({"time":"1","site":"s\r\n1","message":"lf\nonly","note":"mixed\r\nand\nmore"})json" + "\n" + R"json({"time":"2","site":"s\r\n1","message":"carriage\rreturn","note":"\r\n"})json" + "\n" + R"json({"time":"3","site":"s\r\n1","message":"ends\r","note":"line\r\nend"})json" + "\n"; + ASSERT_EQ(write(input, first), 0); + EXPECT_EQ(read(first, "ndjson"), expected_ndjson); + + for (const std::string command : {"cat", "export"}) { + SCOPED_TRACE(command); + const std::string exported = tsfile_cli_test::unique_temp_path( + "tsfile_cli_crlf_" + command, ".csv"); + const std::string second = tsfile_cli_test::unique_temp_path( + "tsfile_cli_crlf_" + command, ".tsfile"); + if (command == "cat") { + const std::string csv = read(first, "csv"); + EXPECT_EQ(csv, expected_csv); + std::ofstream file(exported.c_str(), std::ios::binary); + ASSERT_TRUE(file.good()); + file << csv; + } else { + std::ostringstream out, err; + ASSERT_EQ(tsfile_cli::run_cli({"export", "-t", "t", "--type", "csv", + "-o", exported, first}, + out, err), + 0) + << err.str(); + std::ifstream file(exported.c_str(), std::ios::binary); + ASSERT_TRUE(file.good()); + std::ostringstream bytes; + bytes << file.rdbuf(); + EXPECT_EQ(bytes.str(), expected_csv); + } + ASSERT_EQ(write(exported, second), 0); + EXPECT_EQ(read(second, "ndjson"), expected_ndjson); + std::remove(exported.c_str()); + std::remove(second.c_str()); + } + std::remove(input.c_str()); + std::remove(first.c_str()); +} diff --git a/cpp/test/tools/input_format_test.cc b/cpp/test/tools/input_format_test.cc index ba922ebc1..8e6707973 100644 --- a/cpp/test/tools/input_format_test.cc +++ b/cpp/test/tools/input_format_test.cc @@ -158,16 +158,52 @@ TEST(InputFormatTest, ReadRecordStripsCarriageReturn) { long long n = 0; ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n)); EXPECT_EQ(rec, "a,b"); + EXPECT_EQ(n, 1); + ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n)); + EXPECT_EQ(rec, "c,d"); + EXPECT_EQ(n, 1); + EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n)); +} + +TEST(InputFormatTest, ReadRecordPreservesQuotedLineEndings) { + std::istringstream in( + "1,\"line one\r\n\r\nline \"\"two\"\"\nline three\",x\r\n" + "2,plain,y\r\n"); + std::string rec; + long long n = 0; + ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n)); + EXPECT_EQ(rec, "1,\"line one\r\n\r\nline \"\"two\"\"\nline three\",x"); + EXPECT_EQ(n, 4); + const auto fields = tsfile_cli::split_line(rec, ',', true); + ASSERT_EQ(fields.size(), 3u); + EXPECT_EQ(fields[1], "line one\r\n\r\nline \"two\"\nline three"); + ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n)); + EXPECT_EQ(rec, "2,plain,y"); + EXPECT_EQ(n, 1); + EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n)); +} + +TEST(InputFormatTest, ReadRecordPreservesQuotedCarriageReturnAtEof) { + std::istringstream in("1,\"never closed\r"); + std::string rec; + long long n = 0; + ASSERT_TRUE(tsfile_cli::read_record(in, true, rec, n)); + EXPECT_EQ(rec, "1,\"never closed\r"); + EXPECT_EQ(n, 1); + EXPECT_FALSE(tsfile_cli::read_record(in, true, rec, n)); } TEST(InputFormatTest, ReadRecordTsvIgnoresQuotes) { // With csv_quotes false a quote is just data; no line joining happens. - std::istringstream in("1\t\"open\n2\tclosed\n"); + std::istringstream in("1\t\"open\r\n2\tclosed\r\n"); std::string rec; long long n = 0; ASSERT_TRUE(tsfile_cli::read_record(in, false, rec, n)); EXPECT_EQ(rec, "1\t\"open"); EXPECT_EQ(n, 1); + ASSERT_TRUE(tsfile_cli::read_record(in, false, rec, n)); + EXPECT_EQ(rec, "2\tclosed"); + EXPECT_EQ(n, 1); } TEST(InputFormatTest, ReadRecordUnterminatedQuoteAtEof) { diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index 403b57fa6..5c7a804cc 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -562,7 +562,7 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, return kExitFile; } has_input_stat = true; - fin.open(args.file.c_str()); + fin.open(args.file.c_str(), std::ios::binary); if (!fin.is_open()) { err << "Error: cannot open input: " << args.file << "\n"; return kExitFile; diff --git a/cpp/tools/format/input_format.cc b/cpp/tools/format/input_format.cc index d89368df3..53aee599c 100644 --- a/cpp/tools/format/input_format.cc +++ b/cpp/tools/format/input_format.cc @@ -44,23 +44,26 @@ bool read_record(std::istream& in, bool csv_quotes, std::string& record, std::string physical; bool open_quote = false; while (std::getline(in, physical)) { - strip_cr(physical); ++lines_consumed; - if (!record.empty()) { - record.push_back('\n'); // restore the newline inside the field - } - record += physical; if (csv_quotes) { for (char c : physical) { if (c == '"') { open_quote = !open_quote; } } - if (open_quote) { - continue; // quote still open: the field spans the next line - } } - return true; + if (!open_quote) { + // Strip CR from record boundaries, but keep it as field data + // when the quoted field continues onto the next physical line. + strip_cr(physical); + } + if (!record.empty()) { + record.push_back('\n'); // restore the newline inside the field + } + record += physical; + if (!open_quote) { + return true; + } } return lines_consumed > 0; // trailing record with an unterminated quote } @@ -80,8 +83,7 @@ std::vector split_line(const std::string& line, char delim, } bool normalize_write_columns(const std::vector& specs, - std::vector& out, - std::string& error) { + std::vector& out, std::string& error) { out.clear(); if (specs.empty()) { error = "empty write column declarations"; diff --git a/cpp/tools/format/input_format.h b/cpp/tools/format/input_format.h index 5f76d3fb9..45afd9811 100644 --- a/cpp/tools/format/input_format.h +++ b/cpp/tools/format/input_format.h @@ -38,8 +38,7 @@ struct ColumnDef { bool parse_datatype_name(const std::string& s, common::TSDataType& out); bool normalize_write_columns(const std::vector& specs, - std::vector& out, - std::string& error); + std::vector& out, std::string& error); std::vector split_line(const std::string& line, char delim, bool csv_quotes); bool parse_bool_cell(const std::string& s, bool& out); @@ -49,9 +48,12 @@ bool validate_identifier(const std::string& name, std::string& error); // Read one logical record from `in`. When `csv_quotes` is true a field may span // multiple physical lines if it opens a double-quote that is not yet closed, so -// continuation lines are appended (newline preserved) until the quote closes. -// `lines_consumed` returns how many physical lines were read, so callers can -// keep accurate line numbers. Returns false at end of input with no record. +// continuation lines are appended (LF or CRLF preserved) until the quote +// closes. CRLF outside quotes is a record boundary, not field data. File +// streams should be opened in binary mode to prevent platform newline +// translation. `lines_consumed` returns how many physical lines were read, so +// callers can keep accurate line numbers. Returns false at end of input with no +// record. bool read_record(std::istream& in, bool csv_quotes, std::string& record, long long& lines_consumed); diff --git a/cpp/tools/tools_main.cc b/cpp/tools/tools_main.cc index 785c77cce..6cdb60ac3 100644 --- a/cpp/tools/tools_main.cc +++ b/cpp/tools/tools_main.cc @@ -19,7 +19,12 @@ #include #include -#ifndef _WIN32 +#ifdef _WIN32 +#include +#include + +#include +#else #include #endif #include @@ -29,7 +34,19 @@ #include "cli/run_cli.h" int main(int argc, char** argv) { -#ifndef _WIN32 +#ifdef _WIN32 + // Preserve CSV field bytes in pipes and redirected files. CRT text mode + // otherwise converts CRLF on input and LF on output, including inside + // quotes. + const auto binary_if_redirected = [](FILE* stream) { + const int fd = _fileno(stream); + if (fd >= 0 && _isatty(fd) == 0) { + _setmode(fd, _O_BINARY); + } + }; + binary_if_redirected(stdin); + binary_if_redirected(stdout); +#else std::signal(SIGPIPE, SIG_IGN); #endif std::vector args(argv + 1, argv + argc);