diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index ae70ac386..697e70ef6 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -1877,3 +1877,84 @@ TEST(CliE2E, TableUtf8JsonKeyCollisionsFailInRowsAndStats) { } std::remove(path.c_str()); } + +TEST(CliE2E, WriteCatCsvWritePreservesLeadingBackslashes) { + const std::string input = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_input", ".csv"); + const std::string first = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_first", ".tsfile"); + const std::string exported = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_export", ".csv"); + const std::string second = + tsfile_cli_test::unique_temp_path("tsfile_cli_slash_second", ".tsfile"); + { + std::ofstream csv(input.c_str(), std::ios::binary); + ASSERT_TRUE(csv.good()); + csv << R"csv(time,site,note +1000,s1,\N +2000,s1,"\N" +3000,s1,\leading +4000,s1,back\slash +5000,s1,"\with,comma" +6000,\site,tag\end +7000,\\site,\\N +)csv"; + } + const auto write = [&](const std::string& source, + const std::string& target) { + std::ostringstream out, err; + int code = tsfile_cli::run_cli( + {"write", "--table", "t", "--tag", "site", "STRING", "--field", + "note", "TEXT", "-i", source, "-o", target}, + out, err); + EXPECT_EQ(code, 0) << err.str(); + return code; + }; + ASSERT_EQ(write(input, first), 0); + + std::ostringstream before, before_err, csv_out, csv_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "ndjson", first}, + before, before_err), + 0) + << before_err.str(); + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "csv", first}, + csv_out, csv_err), + 0) + << csv_err.str(); + const std::string csv = csv_out.str(); + EXPECT_NE(csv.find(R"csv(1000,s1,\N +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(2000,s1,\\N +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(3000,s1,\\leading +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(4000,s1,back\slash +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(5000,s1,"\\with,comma" +)csv"), + std::string::npos); + EXPECT_NE(csv.find(R"csv(6000,\\site,tag\end +)csv"), + std::string::npos); + { + std::ofstream file(exported.c_str(), std::ios::binary); + ASSERT_TRUE(file.good()); + file << csv; + } + ASSERT_EQ(write(exported, second), 0); + std::ostringstream after, after_err; + ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "ndjson", second}, + after, after_err), + 0) + << after_err.str(); + EXPECT_EQ(before.str(), after.str()); + + std::remove(input.c_str()); + std::remove(first.c_str()); + std::remove(exported.c_str()); + std::remove(second.c_str()); +} diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index b58fafe3d..59c8cbe29 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -370,3 +370,16 @@ TEST(RowWriterTest, RejectsJsonKeyCollisionsWithoutRows) { EXPECT_FALSE(writer.finish()); EXPECT_TRUE(out.str().empty()); } + +TEST(RowWriterTest, CsvEscapesLeadingBackslashesOnlyInTextValues) { + std::ostringstream out; + RowWriter writer(out, OutputFormat::kCsv, {R"(\header)", "note"}, + {common::STRING, common::TEXT}, false); + ASSERT_TRUE(writer.write({R"(\N)", R"(\a,b)"}, {false, false})); + ASSERT_TRUE(writer.write({R"(back\slash)", ""}, {false, true})); + ASSERT_TRUE(writer.finish()); + EXPECT_EQ(out.str(), R"csv(\header,note +\\N,"\\a,b" +back\slash,\N +)csv"); +} diff --git a/cpp/tools/README.md b/cpp/tools/README.md index 874f52bb0..6c4590fce 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -116,9 +116,11 @@ Shared options: `ndjson` output emits one JSON object per line; numbers/booleans are bare, other values are quoted, nulls are `null`, and non-finite floats become `null`. CSV output follows RFC 4180. -CSV nulls are unquoted `\N`; empty strings are quoted as `""`. Timestamps are raw int64 -values. The `table` format uses a temporary spool to align columns with bounded -memory; prefer `csv`/`ndjson` when temporary disk use is undesirable. `sketch` +CSV nulls are unquoted `\N`; empty strings are quoted as `""`. Non-null STRING/TEXT +values beginning with `\` gain one extra leading `\`, using the same encoding rule +as CSV input described below. Timestamps are raw int64 values. The `table` format +uses a temporary spool to align columns with bounded memory; prefer `csv`/`ndjson` +when temporary disk use is undesirable. `sketch` does not accept `--format`. Text output preserves well-formed UTF-8 and replaces malformed UTF-8 in names @@ -175,8 +177,30 @@ TAG/FIELD of that type; they do not target individual columns and do not affect | `-v, --verbose` | Print a creation summary to stderr after commit (otherwise silent on success) | CSV input uses RFC 4180 quoting with comma separators. A null value is unquoted `\N`; -an empty string is `""`. Header errors, unused or duplicate physical overrides, unknown -types, and incompatible encodings fail before data rows are read. Target-file problems +an empty string is `""`. + +For non-null STRING/TEXT values beginning with a backslash, prepend **one extra +leading backslash** when preparing CSV, including inside quoted cells. `write` +recognizes the unquoted NULL marker first, then removes exactly one leading `\` +from STRING/TEXT cells beginning with `\\`. The decoded literal `\N` remains text. +Quoting a cell does not disable this prefix decoding. + +| CSV cell | Imported value | +|---|---| +| `\N` | NULL | +| `""` | Empty string | +| `\\N` | Literal text `\N` | +| `"\N"` | Literal text `\N` (quoted alternative) | +| `\\path` | Text `\path` with one leading backslash | +| `\\\path` | Text `\\path` with two leading backslashes | +| `"\\a,b"` | Text `\a,b` with one leading backslash and a comma | + +Use the same rule for manually authored CSV and output from `cat -f csv` or +`export --type csv`. Backslashes elsewhere in a value, header names, and non-text +columns are unaffected. + +Header errors, unused or duplicate physical overrides, unknown types, and +incompatible encodings fail before data rows are read. Target-file problems such as an existing output, a missing parent directory, or output equal to input return exit code `3`. diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index 7ee2557f4..d46d681a8 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -202,6 +202,11 @@ void print_command_usage(const std::string& command, std::ostream& os) { "and resolved column physical settings to stderr.\n" "Default: success is silent; target must not exist; CSV header " "must contain time and declared TAG/FIELD names.\n" + R"(CSV values: unquoted \N is NULL; "" is an empty string. +STRING/TEXT: prepend one extra \ to values starting with \, including inside quoted cells. +write removes exactly one leading \ from STRING/TEXT cells starting with \\; backslashes elsewhere are unchanged. +CSV examples: \\N -> literal \N; \\path -> \path; \\\path -> \\path. +)" "Examples:\n" " tsfile-cli write --table sensors --tag site STRING " "--field temperature DOUBLE -i input.csv -o out.tsfile\n"; diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index 33966e494..403b57fa6 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -57,7 +57,8 @@ namespace { struct CsvCell { std::string value; - bool quoted; + bool quoted = false; + bool leading_backslash_prefix_unescaped = false; }; struct DataRow { @@ -395,7 +396,18 @@ std::vector split_csv_cells(const std::string& line, char delim, } bool is_csv_null(const CsvCell& cell) { - return !cell.quoted && cell.value == "\\N"; + return !cell.quoted && !cell.leading_backslash_prefix_unescaped && + cell.value == "\\N"; +} + +void unescape_leading_backslash_prefix(CsvCell& cell) { + if (is_csv_null(cell) || cell.value.size() < 2 || cell.value[0] != '\\' || + cell.value[1] != '\\') { + return; + } + cell.value.erase(0, 1); + // The decoded literal \N must not be mistaken for a NULL marker. + cell.leading_backslash_prefix_unescaped = true; } bool add_typed_value(storage::Tablet& tablet, uint32_t row, @@ -721,6 +733,10 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, r.cells.resize(columns.size()); for (size_t j = 0; j < columns.size(); ++j) { r.cells[j] = fields[field_indexes[j]]; + if (columns[j].type == common::STRING || + columns[j].type == common::TEXT) { + unescape_leading_backslash_prefix(r.cells[j]); + } } std::string device_key; diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index b6ba39c1b..50eee688a 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -524,9 +524,14 @@ bool RowWriter::write(const std::vector& cells, } const common::TSDataType type = i < row_types.size() ? row_types[i] : common::STRING; - const std::string cell = format_cell(type, cells[i]); + std::string cell = format_cell(type, cells[i]); if (fmt_ == OutputFormat::kCsv && cell.empty()) { out_ << "\"\""; + } else if (fmt_ == OutputFormat::kCsv && + (type == common::STRING || type == common::TEXT) && + !cell.empty() && cell[0] == '\\') { + cell.insert(0, 1, '\\'); + out_ << csv_escape(cell); } else { out_ << (fmt_ == OutputFormat::kCsv ? csv_escape(cell) : cell); }