From 466929c78b029bceecefd3704c9be0e2fdf44293 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 30 Sep 2026 10:47:38 +0800 Subject: [PATCH] fix(cpp): escape control characters in tsfile-cli table output The readable 'table' format emitted string cells verbatim, so an embedded newline split one logical record across multiple physical lines and other control characters rendered invisibly. Downstream line-based consumers then misparsed the output with no error. Render backslash, newline, carriage return and tab as the two-character literals \\, \n, \r and \t, and every other non-printable control byte (C0 controls plus DEL) as \uXXXX. CSV and NDJSON output are unchanged. Add unit coverage for the new table_escape() helper (named controls, unicode escapes, DEL boundary, UTF-8 passthrough, column alignment) and a CLI end-to-end test asserting exact table/csv/ndjson output for cells that embed control characters. --- cpp/test/tools/cli_test_util.h | 38 ++++++++++++++++ cpp/test/tools/model_format_e2e_test.cc | 31 +++++++++++++ cpp/test/tools/output_format_test.cc | 59 +++++++++++++++++++++++++ cpp/tools/format/output_format.cc | 48 ++++++++++++++++++-- cpp/tools/format/output_format.h | 1 + 5 files changed, 173 insertions(+), 4 deletions(-) diff --git a/cpp/test/tools/cli_test_util.h b/cpp/test/tools/cli_test_util.h index 7c53f0040..accebc249 100644 --- a/cpp/test/tools/cli_test_util.h +++ b/cpp/test/tools/cli_test_util.h @@ -698,6 +698,44 @@ inline std::string write_empty_field_fixture() { return path; } +// A single-table file whose STRING FIELD values embed control characters +// (newline, tab, backslash). Used to verify that the readable `table` output +// escapes them into visible two-character sequences while CSV/NDJSON keep +// their machine-format semantics. +inline std::string write_control_char_fixture() { + storage::libtsfile_init(); + std::string path = unique_temp_path("tsfile_cli_control_char", ".tsfile"); + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(path, flags, 0666); + auto* schema = new storage::TableSchema( + "t1", + {common::ColumnSchema("site", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("note", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD)}); + auto* writer = new storage::TsFileTableWriter(&file, schema); + const char* sites[] = {"s1", "s2", "s3"}; + const std::string notes[] = {"line1\nline2", "tab\there", "back\\slash"}; + for (int row = 0; row < 3; ++row) { + storage::Tablet tablet( + "t1", {"site", "note"}, {common::STRING, common::STRING}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); + tablet.add_timestamp(0, 1000 + row * 1000); + tablet.add_value(0, "site", sites[row]); + tablet.add_value(0, "note", notes[row]); + writer->write_table(tablet); + } + writer->flush(); + writer->close(); + delete writer; + delete schema; + return path; +} + } // namespace tsfile_cli_test #endif // TSFILE_CLI_TEST_UTIL_H diff --git a/cpp/test/tools/model_format_e2e_test.cc b/cpp/test/tools/model_format_e2e_test.cc index f782ce569..a5cbd9b21 100644 --- a/cpp/test/tools/model_format_e2e_test.cc +++ b/cpp/test/tools/model_format_e2e_test.cc @@ -636,6 +636,37 @@ TEST(IndependentFixtures, SpecialCsvInputsHaveExactCodesAndErrors) { } } +TEST(IndependentFixtures, ControlCharCellsRenderAsVisibleEscapes) { + const std::string fixture = tsfile_cli_test::write_control_char_fixture(); + + // `table` keeps one logical record on one physical line by escaping + // newline, tab and backslash into visible two-character sequences. + expect_cli_exact({"cat", "-t", "t1", "-f", "table", fixture}, 0, + "time site note\n" + "1000 s1 line1\\nline2\n" + "2000 s2 tab\\there\n" + "3000 s3 back\\\\slash\n", + ""); + + // CSV/NDJSON keep their machine-format semantics unchanged: the embedded + // newline/tab/backslash survive verbatim inside CSV quotes or as JSON + // escapes, never flattened into the visible table representation. + expect_cli_exact({"cat", "-t", "t1", "-f", "csv", fixture}, 0, + "time,site,note\n" + "1000,s1,\"line1\nline2\"\n" + "2000,s2,tab\there\n" + "3000,s3,back\\slash\n", + ""); + expect_cli_exact( + {"cat", "-t", "t1", "-f", "ndjson", fixture}, 0, + "{\"time\":\"1000\",\"site\":\"s1\",\"note\":\"line1\\nline2\"}\n" + "{\"time\":\"2000\",\"site\":\"s2\",\"note\":\"tab\\there\"}\n" + "{\"time\":\"3000\",\"site\":\"s3\",\"note\":\"back\\\\slash\"}\n", + ""); + + std::remove(fixture.c_str()); +} + INSTANTIATE_TEST_SUITE_P(TreeAndTable, BothModels, ::testing::Values(ModelFile{true, {"-d", "root.test.d1"}}, diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index c193f464f..55324ff3f 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -93,6 +93,35 @@ TEST(JsonEscapeTest, EscapesQuotesBackslashAndControls) { EXPECT_EQ(tsfile_cli::json_escape("tab\there"), "tab\\there"); } +TEST(TableEscapeTest, EscapesBackslashAndNamedControls) { + EXPECT_EQ(tsfile_cli::table_escape("a\\b"), "a\\\\b"); + EXPECT_EQ(tsfile_cli::table_escape("line\nbreak"), "line\\nbreak"); + EXPECT_EQ(tsfile_cli::table_escape("car\rret"), "car\\rret"); + EXPECT_EQ(tsfile_cli::table_escape("tab\there"), "tab\\there"); +} + +TEST(TableEscapeTest, OtherControlsBecomeUnicodeEscapes) { + EXPECT_EQ(tsfile_cli::table_escape(std::string("a\bb", 3)), "a\\u0008b"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("a\fb", 3)), "a\\u000cb"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("\x01\x1f", 2)), + "\\u0001\\u001f"); +} + +TEST(TableEscapeTest, ControlByteBoundaries) { + // NUL is the lowest C0 control; DEL (0x7f) is the only non-C0 control in + // the C iscntrl() set and must also become \uXXXX rather than pass through. + EXPECT_EQ(tsfile_cli::table_escape(std::string("\x00", 1)), "\\u0000"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("\x7f", 1)), "\\u007f"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("a\177b", 3)), "a\\u007fb"); + + // Space (0x20) is printable: it must survive verbatim, never be escaped. + EXPECT_EQ(tsfile_cli::table_escape("a b"), "a b"); + + // Bytes >= 0x80 are UTF-8 lead/continuation bytes; escaping them would + // corrupt multi-byte sequences, so they must pass through unchanged. + EXPECT_EQ(tsfile_cli::table_escape("\xe4\xb8\xad"), "\xe4\xb8\xad"); +} + TEST(TypeNameTest, KnownTypesMapToNames) { EXPECT_STREQ(tsfile_cli::tsdatatype_name(common::INT64), "INT64"); EXPECT_STREQ(tsfile_cli::tsdatatype_name(common::STRING), "STRING"); @@ -188,6 +217,36 @@ TEST(RowWriterTest, TableAlignsColumns) { "longname BOOLEAN\n"); } +TEST(RowWriterTest, TableEscapesControlCharactersOnOneLine) { + std::ostringstream out; + RowWriter w(out, OutputFormat::kTable, {"time", "note"}, + {common::INT64, common::STRING}, false); + w.write({"1000", "line1\nline2"}, {false, false}); + w.write({"2000", "tab\there"}, {false, false}); + w.write({"3000", "back\\slash"}, {false, false}); + w.finish(); + EXPECT_EQ(out.str(), + "time note\n" + "1000 line1\\nline2\n" + "2000 tab\\there\n" + "3000 back\\\\slash\n"); +} + +TEST(RowWriterTest, TableEscapingKeepsColumnAlignment) { + // When several cells in one row contain controls, each expanded escape + // must still be padded to the same visual width computed at write time. + std::ostringstream out; + RowWriter w(out, OutputFormat::kTable, {"a", "b"}, + {common::STRING, common::STRING}, false); + w.write({"x\ty", "long\nvalue"}, {false, false}); + w.write({"p", "q"}, {false, false}); + w.finish(); + EXPECT_EQ(out.str(), + "a b\n" + "x\\ty long\\nvalue\n" + "p q\n"); +} + TEST(RowWriterTest, ReportsStreamWriteFailure) { FailingStreamBuf buffer; std::ostream out(&buffer); diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index d068591a0..63aa39ad6 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -235,6 +235,45 @@ std::string json_escape(const std::string& s) { return out; } +// The readable `table` format renders control characters as fixed visible +// escapes: backslash, newline, carriage return and tab become the two-character +// literals \\, \n, \r and \t, and every other non-printable control byte +// (C0 controls plus DEL) becomes \uXXXX. This keeps one logical record on one +// physical line regardless of embedded control characters. +std::string table_escape(const std::string& s) { + std::string out; + out.reserve(s.size() + 2); + for (unsigned char c : s) { + switch (c) { + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + // C0 controls (0x00-0x1f) plus DEL (0x7f) are the complete + // set of non-printable control bytes per C iscntrl(). Bytes + // >= 0x80 are UTF-8 continuation/lead bytes and must pass + // through untouched so multi-byte sequences survive. + if (c < 0x20 || c == 0x7f) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } + } + } + return out; +} + namespace { // FLOAT/DOUBLE cells render non-finite values as nan/inf tokens, which have @@ -271,7 +310,7 @@ RowWriter::RowWriter(std::ostream& out, OutputFormat fmt, table_widths_(header_.size(), 0) { if (!no_header_) { for (size_t i = 0; i < header_.size(); ++i) { - table_widths_[i] = header_[i].size(); + table_widths_[i] = table_escape(header_[i]).size(); } } } @@ -360,7 +399,8 @@ bool RowWriter::write(const std::vector& cells, return false; } table_widths_[i] = - std::max(table_widths_[i], format_cell(type, value).size()); + std::max(table_widths_[i], + table_escape(format_cell(type, value)).size()); } return true; } @@ -444,9 +484,9 @@ bool RowWriter::finish() { for (size_t i = 0; i < ncols; ++i) { std::string cell = (i < cells.size() && !(i < nulls.size() && nulls[i])) - ? format_cell( + ? table_escape(format_cell( i < row_types.size() ? row_types[i] : common::STRING, - cells[i]) + cells[i])) : ""; out_ << cell; if (i + 1 < ncols) { diff --git a/cpp/tools/format/output_format.h b/cpp/tools/format/output_format.h index cfab60a69..1a9ed52e9 100644 --- a/cpp/tools/format/output_format.h +++ b/cpp/tools/format/output_format.h @@ -44,6 +44,7 @@ const char* compression_name(common::CompressionType c); std::string csv_escape(const std::string& field); std::string json_escape(const std::string& s); +std::string table_escape(const std::string& s); class RowWriter { public: