From 98d0978c2d8c0ddc2cc3dcc5e5a94e9041b5e370 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 30 Sep 2026 11:23:48 +0800 Subject: [PATCH 1/3] fix(cpp): preserve leading backslashes in CLI CSV round trips --- cpp/test/tools/command_e2e_test.cc | 81 ++++++++++++++++++++++++++++ cpp/test/tools/output_format_test.cc | 13 +++++ cpp/tools/commands/cmd_write.cc | 20 ++++++- cpp/tools/format/output_format.cc | 4 ++ 4 files changed, 116 insertions(+), 2 deletions(-) diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index fae33226d..84561774c 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -1624,3 +1624,84 @@ TEST(CliE2E, WriteDistinguishesCsvNullAndEmptyString) { std::remove(csv.c_str()); std::remove(out_path.c_str()); } + +TEST(CliE2E, WriteCatCsvWritePreservesLeadingBackslashes) { + const std::string input = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_input", ".csv"); + const std::string first = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_first", ".tsfile"); + const std::string exported = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_export", ".csv"); + const std::string second = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_second", ".tsfile"); + { + std::ofstream csv(input.c_str(), std::ios::binary); + ASSERT_TRUE(csv.good()); + csv << R"csv(time,site,note +1000,s1,\N +2000,s1,"\N" +3000,s1,\leading +4000,s1,back\slash +5000,s1,"\with,comma" +6000,\site,tag\end +7000,\\site,\\N +)csv"; + } + const auto write = [&](const std::string& source, + const std::string& target) { + std::ostringstream out, err; + int code = tsfile_cli::run_cli( + {"write", "--table", "t", "--tag", "site", "STRING", "--field", + "note", "TEXT", "-i", source, "-o", target}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + return code; + }; + ASSERT_EQ(write(input, first), 0); + + std::ostringstream before, before_err, csv_out, csv_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "ndjson", first}, + before, before_err), + 0) + << before_err.str(); + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "csv", first}, + csv_out, csv_err), + 0) + << csv_err.str(); + const std::string csv = csv_out.str(); + EXPECT_NE(csv.find(R"csv(1000,s1,\N +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(2000,s1,\\N +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(3000,s1,\\leading +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(4000,s1,back\slash +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(5000,s1,"\\with,comma" +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(6000,\\site,tag\end +)csv"), + std::string::npos); + { + std::ofstream file(exported.c_str(), std::ios::binary); + ASSERT_TRUE(file.good()); + file << csv; + } + ASSERT_EQ(write(exported, second), 0); + std::ostringstream after, after_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "ndjson", second}, + after, after_err), + 0) + << after_err.str(); + EXPECT_EQ(before.str(), after.str()); + + std::remove(input.c_str()); + std::remove(first.c_str()); + std::remove(exported.c_str()); + std::remove(second.c_str()); +} diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index 55324ff3f..473537e2e 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -264,3 +264,16 @@ TEST(RowWriterTest, ReportsFlushFailure) { ASSERT_TRUE(writer.write({"value"}, {false})); EXPECT_FALSE(writer.finish()); } + +TEST(RowWriterTest, CsvEscapesLeadingBackslashesOnlyInTextValues) { + std::ostringstream out; + RowWriter writer(out, OutputFormat::kCsv, {R"(\header)", "note"}, + {common::STRING, common::TEXT}, false); + writer.write({R"(\N)", R"(\a,b)"}, {false, false}); + writer.write({R"(back\slash)", ""}, {false, true}); + ASSERT_TRUE(writer.finish()); + EXPECT_EQ(out.str(), R"csv(\header,note +\\N,"\\a,b" +back\slash,\N +)csv"); +} diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index 33966e494..9ff47cec0 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -57,7 +57,8 @@ namespace { struct CsvCell { std::string value; - bool quoted; + bool quoted = false; + bool leading_backslash_escaped = false; }; struct DataRow { @@ -395,7 +396,18 @@ std::vector split_csv_cells(const std::string& line, char delim, } bool is_csv_null(const CsvCell& cell) { - return !cell.quoted && cell.value == "\\N"; + return !cell.quoted && !cell.leading_backslash_escaped && + cell.value == "\\N"; +} + +void unescape_csv_text(CsvCell& cell) { + if (is_csv_null(cell) || cell.value.size() < 2 || cell.value[0] != '\\' || + cell.value[1] != '\\') { + return; + } + cell.value.erase(0, 1); + // The decoded literal \N must not be mistaken for a NULL marker. + cell.leading_backslash_escaped = true; } bool add_typed_value(storage::Tablet& tablet, uint32_t row, @@ -721,6 +733,10 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, r.cells.resize(columns.size()); for (size_t j = 0; j < columns.size(); ++j) { r.cells[j] = fields[field_indexes[j]]; + if (columns[j].type == common::STRING || + columns[j].type == common::TEXT) { + unescape_csv_text(r.cells[j]); + } } std::string device_key; diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index 63aa39ad6..b724e2ed3 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -454,6 +454,10 @@ bool RowWriter::write(const std::vector& cells, const std::string cell = format_cell(type, cells[i]); if (fmt_ == OutputFormat::kCsv && cell.empty()) { out_ << "\"\""; + } else if (fmt_ == OutputFormat::kCsv && + (type == common::STRING || type == common::TEXT) && + !cell.empty() && cell[0] == '\\') { + out_ << csv_escape(std::string(1, '\\') + cell); } else { out_ << (fmt_ == OutputFormat::kCsv ? csv_escape(cell) : cell); } From d8dec23d60ef1c842353c31b7e7ef5329fc43e43 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 8 Oct 2026 16:26:12 +0800 Subject: [PATCH 2/3] fix(cpp): clarify CLI CSV backslash prefix encoding --- cpp/tools/README.md | 34 ++++++++++++++++++++++++++++----- cpp/tools/cli/run_cli.cc | 7 +++++++ cpp/tools/commands/cmd_write.cc | 10 +++++----- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 37f704896..601bca085 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -116,9 +116,11 @@ Shared options: `ndjson` output emits one JSON object per line; numbers/booleans are bare, other values are quoted, nulls are `null`, and non-finite floats become `null`. CSV output follows RFC 4180. -CSV nulls are unquoted `\N`; empty strings are quoted as `""`. Timestamps are raw int64 -values. The `table` format uses a temporary spool to align columns with bounded -memory; prefer `csv`/`ndjson` when temporary disk use is undesirable. `sketch` +CSV nulls are unquoted `\N`; empty strings are quoted as `""`. Non-null STRING/TEXT +values beginning with `\` gain one extra leading `\`, using the same encoding rule +as CSV input described below. Timestamps are raw int64 values. The `table` format +uses a temporary spool to align columns with bounded memory; prefer `csv`/`ndjson` +when temporary disk use is undesirable. `sketch` does not accept `--format`. ```bash @@ -169,8 +171,30 @@ TAG/FIELD of that type; they do not target individual columns and do not affect | `-v, --verbose` | Print a creation summary to stderr after commit (otherwise silent on success) | CSV input uses RFC 4180 quoting with comma separators. A null value is unquoted `\N`; -an empty string is `""`. Header errors, unused or duplicate physical overrides, unknown -types, and incompatible encodings fail before data rows are read. Target-file problems +an empty string is `""`. + +For non-null STRING/TEXT values beginning with a backslash, prepend **one extra +leading backslash** when preparing CSV, including inside quoted cells. `write` +recognizes the unquoted NULL marker first, then removes exactly one leading `\` +from STRING/TEXT cells beginning with `\\`. The decoded literal `\N` remains text. +Quoting a cell does not disable this prefix decoding. + +| CSV cell | Imported value | +|---|---| +| `\N` | NULL | +| `""` | Empty string | +| `\\N` | Literal text `\N` | +| `"\N"` | Literal text `\N` (quoted alternative) | +| `\\path` | Text `\path` with one leading backslash | +| `\\\path` | Text `\\path` with two leading backslashes | +| `"\\a,b"` | Text `\a,b` with one leading backslash and a comma | + +Use the same rule for manually authored CSV and output from `cat -f csv` or +`export --type csv`. Backslashes elsewhere in a value, header names, and non-text +columns are unaffected. + +Header errors, unused or duplicate physical overrides, unknown types, and +incompatible encodings fail before data rows are read. Target-file problems such as an existing output, a missing parent directory, or output equal to input return exit code `3`. diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index 7ee2557f4..77d58d440 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -202,6 +202,13 @@ void print_command_usage(const std::string& command, std::ostream& os) { "and resolved column physical settings to stderr.\n" "Default: success is silent; target must not exist; CSV header " "must contain time and declared TAG/FIELD names.\n" + "CSV values: unquoted \\N is NULL; \"\" is an empty string.\n" + "STRING/TEXT: prepend one extra \\ to values starting with \\, " + "including inside quoted cells.\n" + "write removes exactly one leading \\ from text cells starting " + "with \\\\; backslashes elsewhere are unchanged.\n" + R"(CSV examples: \\N -> literal \N; \\path -> \path; \\\path -> \\path.)" + "\n" "Examples:\n" " tsfile-cli write --table sensors --tag site STRING " "--field temperature DOUBLE -i input.csv -o out.tsfile\n"; diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index 9ff47cec0..403b57fa6 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -58,7 +58,7 @@ namespace { struct CsvCell { std::string value; bool quoted = false; - bool leading_backslash_escaped = false; + bool leading_backslash_prefix_unescaped = false; }; struct DataRow { @@ -396,18 +396,18 @@ std::vector split_csv_cells(const std::string& line, char delim, } bool is_csv_null(const CsvCell& cell) { - return !cell.quoted && !cell.leading_backslash_escaped && + return !cell.quoted && !cell.leading_backslash_prefix_unescaped && cell.value == "\\N"; } -void unescape_csv_text(CsvCell& cell) { +void unescape_leading_backslash_prefix(CsvCell& cell) { if (is_csv_null(cell) || cell.value.size() < 2 || cell.value[0] != '\\' || cell.value[1] != '\\') { return; } cell.value.erase(0, 1); // The decoded literal \N must not be mistaken for a NULL marker. - cell.leading_backslash_escaped = true; + cell.leading_backslash_prefix_unescaped = true; } bool add_typed_value(storage::Tablet& tablet, uint32_t row, @@ -735,7 +735,7 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, r.cells[j] = fields[field_indexes[j]]; if (columns[j].type == common::STRING || columns[j].type == common::TEXT) { - unescape_csv_text(r.cells[j]); + unescape_leading_backslash_prefix(r.cells[j]); } } From fba28f120c31f9b8ebadd014af28e5bc975c776e Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 8 Oct 2026 16:58:56 +0800 Subject: [PATCH 3/3] fix(cpp): clarify CSV help and check test writes --- cpp/test/tools/output_format_test.cc | 4 ++-- cpp/tools/cli/run_cli.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index 473537e2e..a0a4fb17c 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -269,8 +269,8 @@ TEST(RowWriterTest, CsvEscapesLeadingBackslashesOnlyInTextValues) { std::ostringstream out; RowWriter writer(out, OutputFormat::kCsv, {R"(\header)", "note"}, {common::STRING, common::TEXT}, false); - writer.write({R"(\N)", R"(\a,b)"}, {false, false}); - writer.write({R"(back\slash)", ""}, {false, true}); + ASSERT_TRUE(writer.write({R"(\N)", R"(\a,b)"}, {false, false})); + ASSERT_TRUE(writer.write({R"(back\slash)", ""}, {false, true})); ASSERT_TRUE(writer.finish()); EXPECT_EQ(out.str(), R"csv(\header,note \\N,"\\a,b" diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index 77d58d440..cfafc63c3 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -205,8 +205,8 @@ void print_command_usage(const std::string& command, std::ostream& os) { "CSV values: unquoted \\N is NULL; \"\" is an empty string.\n" "STRING/TEXT: prepend one extra \\ to values starting with \\, " "including inside quoted cells.\n" - "write removes exactly one leading \\ from text cells starting " - "with \\\\; backslashes elsewhere are unchanged.\n" + "write removes exactly one leading \\ from STRING/TEXT cells " + "starting with \\\\; backslashes elsewhere are unchanged.\n" R"(CSV examples: \\N -> literal \N; \\path -> \path; \\\path -> \\path.)" "\n" "Examples:\n"