Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions cpp/test/tools/cli_test_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,44 @@ inline std::string write_empty_field_fixture() {
return path;
}

// A single-table file whose STRING FIELD values embed control characters
// (newline, tab, backslash). Used to verify that the readable `table` output
// escapes them into visible two-character sequences while CSV/NDJSON keep
// their machine-format semantics.
inline std::string write_control_char_fixture() {
storage::libtsfile_init();
std::string path = unique_temp_path("tsfile_cli_control_char", ".tsfile");
storage::WriteFile file;
int flags = O_WRONLY | O_CREAT | O_TRUNC;
#ifdef _WIN32
flags |= O_BINARY;
#endif
file.create(path, flags, 0666);
auto* schema = new storage::TableSchema(
"t1",
{common::ColumnSchema("site", common::STRING, common::UNCOMPRESSED,
common::PLAIN, common::ColumnCategory::TAG),
common::ColumnSchema("note", common::STRING, common::UNCOMPRESSED,
common::PLAIN, common::ColumnCategory::FIELD)});
auto* writer = new storage::TsFileTableWriter(&file, schema);
const char* sites[] = {"s1", "s2", "s3"};
const std::string notes[] = {"line1\nline2", "tab\there", "back\\slash"};
for (int row = 0; row < 3; ++row) {
storage::Tablet tablet(
"t1", {"site", "note"}, {common::STRING, common::STRING},
{common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1);
tablet.add_timestamp(0, 1000 + row * 1000);
tablet.add_value(0, "site", sites[row]);
tablet.add_value(0, "note", notes[row]);
writer->write_table(tablet);
}
writer->flush();
writer->close();
delete writer;
delete schema;
return path;
}

} // namespace tsfile_cli_test

#endif // TSFILE_CLI_TEST_UTIL_H
31 changes: 31 additions & 0 deletions cpp/test/tools/model_format_e2e_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,37 @@ TEST(IndependentFixtures, SpecialCsvInputsHaveExactCodesAndErrors) {
}
}

TEST(IndependentFixtures, ControlCharCellsRenderAsVisibleEscapes) {
const std::string fixture = tsfile_cli_test::write_control_char_fixture();

// `table` keeps one logical record on one physical line by escaping
// newline, tab and backslash into visible two-character sequences.
expect_cli_exact({"cat", "-t", "t1", "-f", "table", fixture}, 0,
"time site note\n"
"1000 s1 line1\\nline2\n"
"2000 s2 tab\\there\n"
"3000 s3 back\\\\slash\n",
"");

// CSV/NDJSON keep their machine-format semantics unchanged: the embedded
// newline/tab/backslash survive verbatim inside CSV quotes or as JSON
// escapes, never flattened into the visible table representation.
expect_cli_exact({"cat", "-t", "t1", "-f", "csv", fixture}, 0,
"time,site,note\n"
"1000,s1,\"line1\nline2\"\n"
"2000,s2,tab\there\n"
"3000,s3,back\\slash\n",
"");
expect_cli_exact(
{"cat", "-t", "t1", "-f", "ndjson", fixture}, 0,
"{\"time\":\"1000\",\"site\":\"s1\",\"note\":\"line1\\nline2\"}\n"
"{\"time\":\"2000\",\"site\":\"s2\",\"note\":\"tab\\there\"}\n"
"{\"time\":\"3000\",\"site\":\"s3\",\"note\":\"back\\\\slash\"}\n",
"");

std::remove(fixture.c_str());
}

INSTANTIATE_TEST_SUITE_P(TreeAndTable, BothModels,
::testing::Values(ModelFile{true,
{"-d", "root.test.d1"}},
Expand Down
59 changes: 59 additions & 0 deletions cpp/test/tools/output_format_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,35 @@ TEST(JsonEscapeTest, EscapesQuotesBackslashAndControls) {
EXPECT_EQ(tsfile_cli::json_escape("tab\there"), "tab\\there");
}

TEST(TableEscapeTest, EscapesBackslashAndNamedControls) {
EXPECT_EQ(tsfile_cli::table_escape("a\\b"), "a\\\\b");
EXPECT_EQ(tsfile_cli::table_escape("line\nbreak"), "line\\nbreak");
EXPECT_EQ(tsfile_cli::table_escape("car\rret"), "car\\rret");
EXPECT_EQ(tsfile_cli::table_escape("tab\there"), "tab\\there");
}

TEST(TableEscapeTest, OtherControlsBecomeUnicodeEscapes) {
EXPECT_EQ(tsfile_cli::table_escape(std::string("a\bb", 3)), "a\\u0008b");
EXPECT_EQ(tsfile_cli::table_escape(std::string("a\fb", 3)), "a\\u000cb");
EXPECT_EQ(tsfile_cli::table_escape(std::string("\x01\x1f", 2)),
"\\u0001\\u001f");
}

TEST(TableEscapeTest, ControlByteBoundaries) {
// NUL is the lowest C0 control; DEL (0x7f) is the only non-C0 control in
// the C iscntrl() set and must also become \uXXXX rather than pass through.
EXPECT_EQ(tsfile_cli::table_escape(std::string("\x00", 1)), "\\u0000");
EXPECT_EQ(tsfile_cli::table_escape(std::string("\x7f", 1)), "\\u007f");
EXPECT_EQ(tsfile_cli::table_escape(std::string("a\177b", 3)), "a\\u007fb");

// Space (0x20) is printable: it must survive verbatim, never be escaped.
EXPECT_EQ(tsfile_cli::table_escape("a b"), "a b");

// Bytes >= 0x80 are UTF-8 lead/continuation bytes; escaping them would
// corrupt multi-byte sequences, so they must pass through unchanged.
EXPECT_EQ(tsfile_cli::table_escape("\xe4\xb8\xad"), "\xe4\xb8\xad");
}

TEST(TypeNameTest, KnownTypesMapToNames) {
EXPECT_STREQ(tsfile_cli::tsdatatype_name(common::INT64), "INT64");
EXPECT_STREQ(tsfile_cli::tsdatatype_name(common::STRING), "STRING");
Expand Down Expand Up @@ -188,6 +217,36 @@ TEST(RowWriterTest, TableAlignsColumns) {
"longname BOOLEAN\n");
}

TEST(RowWriterTest, TableEscapesControlCharactersOnOneLine) {
std::ostringstream out;
RowWriter w(out, OutputFormat::kTable, {"time", "note"},
{common::INT64, common::STRING}, false);
w.write({"1000", "line1\nline2"}, {false, false});
w.write({"2000", "tab\there"}, {false, false});
w.write({"3000", "back\\slash"}, {false, false});
w.finish();
EXPECT_EQ(out.str(),
"time note\n"
"1000 line1\\nline2\n"
"2000 tab\\there\n"
"3000 back\\\\slash\n");
}

TEST(RowWriterTest, TableEscapingKeepsColumnAlignment) {
// When several cells in one row contain controls, each expanded escape
// must still be padded to the same visual width computed at write time.
std::ostringstream out;
RowWriter w(out, OutputFormat::kTable, {"a", "b"},
{common::STRING, common::STRING}, false);
w.write({"x\ty", "long\nvalue"}, {false, false});
w.write({"p", "q"}, {false, false});
w.finish();
EXPECT_EQ(out.str(),
"a b\n"
"x\\ty long\\nvalue\n"
"p q\n");
}

TEST(RowWriterTest, ReportsStreamWriteFailure) {
FailingStreamBuf buffer;
std::ostream out(&buffer);
Expand Down
48 changes: 44 additions & 4 deletions cpp/tools/format/output_format.cc
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,45 @@ std::string json_escape(const std::string& s) {
return out;
}

// The readable `table` format renders control characters as fixed visible
// escapes: backslash, newline, carriage return and tab become the two-character
// literals \\, \n, \r and \t, and every other non-printable control byte
// (C0 controls plus DEL) becomes \uXXXX. This keeps one logical record on one
// physical line regardless of embedded control characters.
std::string table_escape(const std::string& s) {
std::string out;
out.reserve(s.size() + 2);
for (unsigned char c : s) {
switch (c) {
case '\\':
out += "\\\\";
break;
case '\n':
out += "\\n";
break;
case '\r':
out += "\\r";
break;
case '\t':
out += "\\t";
break;
default:
// C0 controls (0x00-0x1f) plus DEL (0x7f) are the complete
// set of non-printable control bytes per C iscntrl(). Bytes
// >= 0x80 are UTF-8 continuation/lead bytes and must pass
// through untouched so multi-byte sequences survive.
if (c < 0x20 || c == 0x7f) {
char buf[8];
std::snprintf(buf, sizeof(buf), "\\u%04x", c);
out += buf;
} else {
out += static_cast<char>(c);
}
}
}
return out;
}

namespace {

// FLOAT/DOUBLE cells render non-finite values as nan/inf tokens, which have
Expand Down Expand Up @@ -271,7 +310,7 @@ RowWriter::RowWriter(std::ostream& out, OutputFormat fmt,
table_widths_(header_.size(), 0) {
if (!no_header_) {
for (size_t i = 0; i < header_.size(); ++i) {
table_widths_[i] = header_[i].size();
table_widths_[i] = table_escape(header_[i]).size();
}
}
}
Expand Down Expand Up @@ -360,7 +399,8 @@ bool RowWriter::write(const std::vector<std::string>& cells,
return false;
}
table_widths_[i] =
std::max(table_widths_[i], format_cell(type, value).size());
std::max(table_widths_[i],
table_escape(format_cell(type, value)).size());
}
return true;
}
Expand Down Expand Up @@ -444,9 +484,9 @@ bool RowWriter::finish() {
for (size_t i = 0; i < ncols; ++i) {
std::string cell =
(i < cells.size() && !(i < nulls.size() && nulls[i]))
? format_cell(
? table_escape(format_cell(
i < row_types.size() ? row_types[i] : common::STRING,
cells[i])
cells[i]))
: "";
out_ << cell;
if (i + 1 < ncols) {
Expand Down
1 change: 1 addition & 0 deletions cpp/tools/format/output_format.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ const char* compression_name(common::CompressionType c);

std::string csv_escape(const std::string& field);
std::string json_escape(const std::string& s);
std::string table_escape(const std::string& s);

class RowWriter {
public:
Expand Down
Loading