diff --git a/cpp/test/tools/cli_test_util.h b/cpp/test/tools/cli_test_util.h index 7c53f0040..accebc249 100644 --- a/cpp/test/tools/cli_test_util.h +++ b/cpp/test/tools/cli_test_util.h @@ -698,6 +698,44 @@ inline std::string write_empty_field_fixture() { return path; } +// A single-table file whose STRING FIELD values embed control characters +// (newline, tab, backslash). Used to verify that the readable `table` output +// escapes them into visible two-character sequences while CSV/NDJSON keep +// their machine-format semantics. +inline std::string write_control_char_fixture() { + storage::libtsfile_init(); + std::string path = unique_temp_path("tsfile_cli_control_char", ".tsfile"); + storage::WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + file.create(path, flags, 0666); + auto* schema = new storage::TableSchema( + "t1", + {common::ColumnSchema("site", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::TAG), + common::ColumnSchema("note", common::STRING, common::UNCOMPRESSED, + common::PLAIN, common::ColumnCategory::FIELD)}); + auto* writer = new storage::TsFileTableWriter(&file, schema); + const char* sites[] = {"s1", "s2", "s3"}; + const std::string notes[] = {"line1\nline2", "tab\there", "back\\slash"}; + for (int row = 0; row < 3; ++row) { + storage::Tablet tablet( + "t1", {"site", "note"}, {common::STRING, common::STRING}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD}, 1); + tablet.add_timestamp(0, 1000 + row * 1000); + tablet.add_value(0, "site", sites[row]); + tablet.add_value(0, "note", notes[row]); + writer->write_table(tablet); + } + writer->flush(); + writer->close(); + delete writer; + delete schema; + return path; +} + } // namespace tsfile_cli_test #endif // TSFILE_CLI_TEST_UTIL_H diff --git a/cpp/test/tools/model_format_e2e_test.cc b/cpp/test/tools/model_format_e2e_test.cc index 11d20d879..8e0825b2d 100644 --- a/cpp/test/tools/model_format_e2e_test.cc +++ b/cpp/test/tools/model_format_e2e_test.cc @@ -660,6 +660,37 @@ TEST(IndependentFixtures, SpecialCsvInputsHaveExactCodesAndErrors) { } } +TEST(IndependentFixtures, ControlCharCellsRenderAsVisibleEscapes) { + const std::string fixture = tsfile_cli_test::write_control_char_fixture(); + + // `table` keeps one logical record on one physical line by escaping + // newline, tab and backslash into visible two-character sequences. + expect_cli_exact({"cat", "-t", "t1", "-f", "table", fixture}, 0, + "time site note\n" + "1000 s1 line1\\nline2\n" + "2000 s2 tab\\there\n" + "3000 s3 back\\\\slash\n", + ""); + + // CSV/NDJSON keep their machine-format semantics unchanged: the embedded + // newline/tab/backslash survive verbatim inside CSV quotes or as JSON + // escapes, never flattened into the visible table representation. + expect_cli_exact({"cat", "-t", "t1", "-f", "csv", fixture}, 0, + "time,site,note\n" + "1000,s1,\"line1\nline2\"\n" + "2000,s2,tab\there\n" + "3000,s3,back\\slash\n", + ""); + expect_cli_exact( + {"cat", "-t", "t1", "-f", "ndjson", fixture}, 0, + "{\"time\":\"1000\",\"site\":\"s1\",\"note\":\"line1\\nline2\"}\n" + "{\"time\":\"2000\",\"site\":\"s2\",\"note\":\"tab\\there\"}\n" + "{\"time\":\"3000\",\"site\":\"s3\",\"note\":\"back\\\\slash\"}\n", + ""); + + std::remove(fixture.c_str()); +} + INSTANTIATE_TEST_SUITE_P(TreeAndTable, BothModels, ::testing::Values(ModelFile{true, {"-d", "root.test.d1"}}, diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index c193f464f..55324ff3f 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -93,6 +93,35 @@ TEST(JsonEscapeTest, EscapesQuotesBackslashAndControls) { EXPECT_EQ(tsfile_cli::json_escape("tab\there"), "tab\\there"); } +TEST(TableEscapeTest, EscapesBackslashAndNamedControls) { + EXPECT_EQ(tsfile_cli::table_escape("a\\b"), "a\\\\b"); + EXPECT_EQ(tsfile_cli::table_escape("line\nbreak"), "line\\nbreak"); + EXPECT_EQ(tsfile_cli::table_escape("car\rret"), "car\\rret"); + EXPECT_EQ(tsfile_cli::table_escape("tab\there"), "tab\\there"); +} + +TEST(TableEscapeTest, OtherControlsBecomeUnicodeEscapes) { + EXPECT_EQ(tsfile_cli::table_escape(std::string("a\bb", 3)), "a\\u0008b"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("a\fb", 3)), "a\\u000cb"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("\x01\x1f", 2)), + "\\u0001\\u001f"); +} + +TEST(TableEscapeTest, ControlByteBoundaries) { + // NUL is the lowest C0 control; DEL (0x7f) is the only non-C0 control in + // the C iscntrl() set and must also become \uXXXX rather than pass through. + EXPECT_EQ(tsfile_cli::table_escape(std::string("\x00", 1)), "\\u0000"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("\x7f", 1)), "\\u007f"); + EXPECT_EQ(tsfile_cli::table_escape(std::string("a\177b", 3)), "a\\u007fb"); + + // Space (0x20) is printable: it must survive verbatim, never be escaped. + EXPECT_EQ(tsfile_cli::table_escape("a b"), "a b"); + + // Bytes >= 0x80 are UTF-8 lead/continuation bytes; escaping them would + // corrupt multi-byte sequences, so they must pass through unchanged. + EXPECT_EQ(tsfile_cli::table_escape("\xe4\xb8\xad"), "\xe4\xb8\xad"); +} + TEST(TypeNameTest, KnownTypesMapToNames) { EXPECT_STREQ(tsfile_cli::tsdatatype_name(common::INT64), "INT64"); EXPECT_STREQ(tsfile_cli::tsdatatype_name(common::STRING), "STRING"); @@ -188,6 +217,36 @@ TEST(RowWriterTest, TableAlignsColumns) { "longname BOOLEAN\n"); } +TEST(RowWriterTest, TableEscapesControlCharactersOnOneLine) { + std::ostringstream out; + RowWriter w(out, OutputFormat::kTable, {"time", "note"}, + {common::INT64, common::STRING}, false); + w.write({"1000", "line1\nline2"}, {false, false}); + w.write({"2000", "tab\there"}, {false, false}); + w.write({"3000", "back\\slash"}, {false, false}); + w.finish(); + EXPECT_EQ(out.str(), + "time note\n" + "1000 line1\\nline2\n" + "2000 tab\\there\n" + "3000 back\\\\slash\n"); +} + +TEST(RowWriterTest, TableEscapingKeepsColumnAlignment) { + // When several cells in one row contain controls, each expanded escape + // must still be padded to the same visual width computed at write time. + std::ostringstream out; + RowWriter w(out, OutputFormat::kTable, {"a", "b"}, + {common::STRING, common::STRING}, false); + w.write({"x\ty", "long\nvalue"}, {false, false}); + w.write({"p", "q"}, {false, false}); + w.finish(); + EXPECT_EQ(out.str(), + "a b\n" + "x\\ty long\\nvalue\n" + "p q\n"); +} + TEST(RowWriterTest, ReportsStreamWriteFailure) { FailingStreamBuf buffer; std::ostream out(&buffer); diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index d068591a0..63aa39ad6 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -235,6 +235,45 @@ std::string json_escape(const std::string& s) { return out; } +// The readable `table` format renders control characters as fixed visible +// escapes: backslash, newline, carriage return and tab become the two-character +// literals \\, \n, \r and \t, and every other non-printable control byte +// (C0 controls plus DEL) becomes \uXXXX. This keeps one logical record on one +// physical line regardless of embedded control characters. +std::string table_escape(const std::string& s) { + std::string out; + out.reserve(s.size() + 2); + for (unsigned char c : s) { + switch (c) { + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + // C0 controls (0x00-0x1f) plus DEL (0x7f) are the complete + // set of non-printable control bytes per C iscntrl(). Bytes + // >= 0x80 are UTF-8 continuation/lead bytes and must pass + // through untouched so multi-byte sequences survive. + if (c < 0x20 || c == 0x7f) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } + } + } + return out; +} + namespace { // FLOAT/DOUBLE cells render non-finite values as nan/inf tokens, which have @@ -271,7 +310,7 @@ RowWriter::RowWriter(std::ostream& out, OutputFormat fmt, table_widths_(header_.size(), 0) { if (!no_header_) { for (size_t i = 0; i < header_.size(); ++i) { - table_widths_[i] = header_[i].size(); + table_widths_[i] = table_escape(header_[i]).size(); } } } @@ -360,7 +399,8 @@ bool RowWriter::write(const std::vector& cells, return false; } table_widths_[i] = - std::max(table_widths_[i], format_cell(type, value).size()); + std::max(table_widths_[i], + table_escape(format_cell(type, value)).size()); } return true; } @@ -444,9 +484,9 @@ bool RowWriter::finish() { for (size_t i = 0; i < ncols; ++i) { std::string cell = (i < cells.size() && !(i < nulls.size() && nulls[i])) - ? format_cell( + ? table_escape(format_cell( i < row_types.size() ? row_types[i] : common::STRING, - cells[i]) + cells[i])) : ""; out_ << cell; if (i + 1 < ncols) { diff --git a/cpp/tools/format/output_format.h b/cpp/tools/format/output_format.h index cfab60a69..1a9ed52e9 100644 --- a/cpp/tools/format/output_format.h +++ b/cpp/tools/format/output_format.h @@ -44,6 +44,7 @@ const char* compression_name(common::CompressionType c); std::string csv_escape(const std::string& field); std::string json_escape(const std::string& s); +std::string table_escape(const std::string& s); class RowWriter { public: