Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 81 additions & 0 deletions cpp/test/tools/command_e2e_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1877,3 +1877,84 @@ TEST(CliE2E, TableUtf8JsonKeyCollisionsFailInRowsAndStats) {
}
std::remove(path.c_str());
}

TEST(CliE2E, WriteCatCsvWritePreservesLeadingBackslashes) {
const std::string input =
tsfile_cli_test::unique_temp_path("tsfile_cli_slash_input", ".csv");
const std::string first =
tsfile_cli_test::unique_temp_path("tsfile_cli_slash_first", ".tsfile");
const std::string exported =
tsfile_cli_test::unique_temp_path("tsfile_cli_slash_export", ".csv");
const std::string second =
tsfile_cli_test::unique_temp_path("tsfile_cli_slash_second", ".tsfile");
{
std::ofstream csv(input.c_str(), std::ios::binary);
ASSERT_TRUE(csv.good());
csv << R"csv(time,site,note
1000,s1,\N
2000,s1,"\N"
3000,s1,\leading
4000,s1,back\slash
5000,s1,"\with,comma"
6000,\site,tag\end
7000,\\site,\\N
)csv";
}
const auto write = [&](const std::string& source,
const std::string& target) {
std::ostringstream out, err;
int code = tsfile_cli::run_cli(
{"write", "--table", "t", "--tag", "site", "STRING", "--field",
"note", "TEXT", "-i", source, "-o", target},
out, err);
EXPECT_EQ(code, 0) << err.str();
return code;
};
ASSERT_EQ(write(input, first), 0);

std::ostringstream before, before_err, csv_out, csv_err;
ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "ndjson", first},
before, before_err),
0)
<< before_err.str();
ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "csv", first},
csv_out, csv_err),
0)
<< csv_err.str();
const std::string csv = csv_out.str();
EXPECT_NE(csv.find(R"csv(1000,s1,\N
)csv"),
std::string::npos);
EXPECT_NE(csv.find(R"csv(2000,s1,\\N
)csv"),
std::string::npos);
EXPECT_NE(csv.find(R"csv(3000,s1,\\leading
)csv"),
std::string::npos);
EXPECT_NE(csv.find(R"csv(4000,s1,back\slash
)csv"),
std::string::npos);
EXPECT_NE(csv.find(R"csv(5000,s1,"\\with,comma"
)csv"),
std::string::npos);
EXPECT_NE(csv.find(R"csv(6000,\\site,tag\end
)csv"),
std::string::npos);
{
std::ofstream file(exported.c_str(), std::ios::binary);
ASSERT_TRUE(file.good());
file << csv;
}
ASSERT_EQ(write(exported, second), 0);
std::ostringstream after, after_err;
ASSERT_EQ(tsfile_cli::run_cli({"cat", "-t", "t", "-f", "ndjson", second},
after, after_err),
0)
<< after_err.str();
EXPECT_EQ(before.str(), after.str());

std::remove(input.c_str());
std::remove(first.c_str());
std::remove(exported.c_str());
std::remove(second.c_str());
}
13 changes: 13 additions & 0 deletions cpp/test/tools/output_format_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -370,3 +370,16 @@ TEST(RowWriterTest, RejectsJsonKeyCollisionsWithoutRows) {
EXPECT_FALSE(writer.finish());
EXPECT_TRUE(out.str().empty());
}

TEST(RowWriterTest, CsvEscapesLeadingBackslashesOnlyInTextValues) {
std::ostringstream out;
RowWriter writer(out, OutputFormat::kCsv, {R"(\header)", "note"},
{common::STRING, common::TEXT}, false);
ASSERT_TRUE(writer.write({R"(\N)", R"(\a,b)"}, {false, false}));
ASSERT_TRUE(writer.write({R"(back\slash)", ""}, {false, true}));
ASSERT_TRUE(writer.finish());
EXPECT_EQ(out.str(), R"csv(\header,note
\\N,"\\a,b"
back\slash,\N
)csv");
}
34 changes: 29 additions & 5 deletions cpp/tools/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,11 @@ Shared options:

`ndjson` output emits one JSON object per line; numbers/booleans are bare, other values are
quoted, nulls are `null`, and non-finite floats become `null`. CSV output follows RFC 4180.
CSV nulls are unquoted `\N`; empty strings are quoted as `""`. Timestamps are raw int64
values. The `table` format uses a temporary spool to align columns with bounded
memory; prefer `csv`/`ndjson` when temporary disk use is undesirable. `sketch`
CSV nulls are unquoted `\N`; empty strings are quoted as `""`. Non-null STRING/TEXT
values beginning with `\` gain one extra leading `\`, using the same encoding rule
as CSV input described below. Timestamps are raw int64 values. The `table` format
uses a temporary spool to align columns with bounded memory; prefer `csv`/`ndjson`
when temporary disk use is undesirable. `sketch`
does not accept `--format`.

Text output preserves well-formed UTF-8 and replaces malformed UTF-8 in names
Expand Down Expand Up @@ -175,8 +177,30 @@ TAG/FIELD of that type; they do not target individual columns and do not affect
| `-v, --verbose` | Print a creation summary to stderr after commit (otherwise silent on success) |

CSV input uses RFC 4180 quoting with comma separators. A null value is unquoted `\N`;
an empty string is `""`. Header errors, unused or duplicate physical overrides, unknown
types, and incompatible encodings fail before data rows are read. Target-file problems
an empty string is `""`.

For non-null STRING/TEXT values beginning with a backslash, prepend **one extra
leading backslash** when preparing CSV, including inside quoted cells. `write`
recognizes the unquoted NULL marker first, then removes exactly one leading `\`
from STRING/TEXT cells beginning with `\\`. The decoded literal `\N` remains text.
Quoting a cell does not disable this prefix decoding.

| CSV cell | Imported value |
|---|---|
| `\N` | NULL |
| `""` | Empty string |
| `\\N` | Literal text `\N` |
| `"\N"` | Literal text `\N` (quoted alternative) |
| `\\path` | Text `\path` with one leading backslash |
| `\\\path` | Text `\\path` with two leading backslashes |
| `"\\a,b"` | Text `\a,b` with one leading backslash and a comma |

Use the same rule for manually authored CSV and output from `cat -f csv` or
`export --type csv`. Backslashes elsewhere in a value, header names, and non-text
columns are unaffected.

Header errors, unused or duplicate physical overrides, unknown types, and
incompatible encodings fail before data rows are read. Target-file problems
such as an existing output, a missing parent directory, or output equal to input return
exit code `3`.

Expand Down
5 changes: 5 additions & 0 deletions cpp/tools/cli/run_cli.cc
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,11 @@ void print_command_usage(const std::string& command, std::ostream& os) {
"and resolved column physical settings to stderr.\n"
"Default: success is silent; target must not exist; CSV header "
"must contain time and declared TAG/FIELD names.\n"
R"(CSV values: unquoted \N is NULL; "" is an empty string.
STRING/TEXT: prepend one extra \ to values starting with \, including inside quoted cells.
write removes exactly one leading \ from STRING/TEXT cells starting with \\; backslashes elsewhere are unchanged.
CSV examples: \\N -> literal \N; \\path -> \path; \\\path -> \\path.
)"
"Examples:\n"
" tsfile-cli write --table sensors --tag site STRING "
"--field temperature DOUBLE -i input.csv -o out.tsfile\n";
Expand Down
20 changes: 18 additions & 2 deletions cpp/tools/commands/cmd_write.cc
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ namespace {

struct CsvCell {
std::string value;
bool quoted;
bool quoted = false;
bool leading_backslash_prefix_unescaped = false;
};

struct DataRow {
Expand Down Expand Up @@ -395,7 +396,18 @@ std::vector<CsvCell> split_csv_cells(const std::string& line, char delim,
}

bool is_csv_null(const CsvCell& cell) {
return !cell.quoted && cell.value == "\\N";
return !cell.quoted && !cell.leading_backslash_prefix_unescaped &&
cell.value == "\\N";
}

void unescape_leading_backslash_prefix(CsvCell& cell) {
if (is_csv_null(cell) || cell.value.size() < 2 || cell.value[0] != '\\' ||
cell.value[1] != '\\') {
return;
}
cell.value.erase(0, 1);
// The decoded literal \N must not be mistaken for a NULL marker.
cell.leading_backslash_prefix_unescaped = true;
}
Comment thread
ColinLeeo marked this conversation as resolved.

bool add_typed_value(storage::Tablet& tablet, uint32_t row,
Expand Down Expand Up @@ -721,6 +733,10 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/,
r.cells.resize(columns.size());
for (size_t j = 0; j < columns.size(); ++j) {
r.cells[j] = fields[field_indexes[j]];
if (columns[j].type == common::STRING ||
columns[j].type == common::TEXT) {
unescape_leading_backslash_prefix(r.cells[j]);
}
}

std::string device_key;
Expand Down
7 changes: 6 additions & 1 deletion cpp/tools/format/output_format.cc
Original file line number Diff line number Diff line change
Expand Up @@ -524,9 +524,14 @@ bool RowWriter::write(const std::vector<std::string>& cells,
}
const common::TSDataType type =
i < row_types.size() ? row_types[i] : common::STRING;
const std::string cell = format_cell(type, cells[i]);
std::string cell = format_cell(type, cells[i]);
if (fmt_ == OutputFormat::kCsv && cell.empty()) {
out_ << "\"\"";
} else if (fmt_ == OutputFormat::kCsv &&
(type == common::STRING || type == common::TEXT) &&
!cell.empty() && cell[0] == '\\') {
cell.insert(0, 1, '\\');
out_ << csv_escape(cell);
} else {
out_ << (fmt_ == OutputFormat::kCsv ? csv_escape(cell) : cell);
}
Expand Down
Loading