You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by co...@apache.org on 2026/10/08 15:01:43 UTC
(tsfile) branch develop updated: fix(cpp): replace malformed UTF-8 in CLI text output (#997)
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 133c9b63b fix(cpp): replace malformed UTF-8 in CLI text output (#997)
133c9b63b is described below
commit 133c9b63ba52fd4fcbdad0a30f17fd188ddb5895
Author: Colin Lee <10...@users.noreply.github.com>
AuthorDate: Thu Oct 8 23:01:38 2026 +0800
fix(cpp): replace malformed UTF-8 in CLI text output (#997)
* fix(cpp): replace malformed UTF-8 in CLI text output
* fix(cpp): reject colliding JSON keys after UTF-8 replacement
---
cpp/test/tools/command_e2e_test.cc | 253 ++++++++++++++++++++++++++++++++++
cpp/test/tools/output_format_test.cc | 110 ++++++++++++++-
cpp/tools/README.md | 6 +
cpp/tools/commands/cmd_export.cc | 27 +---
cpp/tools/commands/cmd_sketch.cc | 21 +--
cpp/tools/commands/cmd_stats.cc | 4 +
cpp/tools/commands/row_query.cc | 14 +-
cpp/tools/format/output_format.cc | 95 +++++++++++--
cpp/tools/format/output_format.h | 7 +
cpp/tools/format/result_set_format.cc | 11 +-
cpp/tools/format/result_set_format.h | 4 +-
11 files changed, 493 insertions(+), 59 deletions(-)
diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc
index fae33226d..ae70ac386 100644
--- a/cpp/test/tools/command_e2e_test.cc
+++ b/cpp/test/tools/command_e2e_test.cc
@@ -24,8 +24,13 @@
#include <sstream>
#include <string>
+#ifdef _WIN32
+#include <direct.h>
+#endif
+
#include "cli/run_cli.h"
#include "cli_test_util.h"
+#include "format/input_format.h"
namespace {
@@ -1624,3 +1629,251 @@ TEST(CliE2E, WriteDistinguishesCsvNullAndEmptyString) {
std::remove(csv.c_str());
std::remove(out_path.c_str());
}
+
+TEST(CliE2E, InvalidUtf8IdentifiersAreReplacedOnlyInOutput) {
+ storage::libtsfile_init();
+ const std::string path =
+ tsfile_cli_test::unique_temp_path("tsfile_cli_utf8_names", ".tsfile");
+ std::string device = "root.device\xff";
+ std::string valid_device = "root.\xe4\xb8\xad\xe6\x96\x87";
+ const std::string measurement = "value\xe1\x80";
+ const std::string valid_measurement = "\xe6\xb8\xa9\xe5\xba\xa6";
+ const std::string replacement = "\xef\xbf\xbd";
+ {
+ storage::WriteFile file;
+ int flags = O_WRONLY | O_CREAT | O_TRUNC;
+#ifdef _WIN32
+ flags |= O_BINARY;
+#endif
+ ASSERT_EQ(file.create(path, flags, 0666), common::E_OK);
+ storage::MeasurementSchema schema(measurement, common::INT64,
+ common::PLAIN, common::UNCOMPRESSED);
+ storage::MeasurementSchema valid_schema(valid_measurement,
+ common::INT64, common::PLAIN,
+ common::UNCOMPRESSED);
+ storage::TsFileTreeWriter writer(&file);
+ ASSERT_EQ(writer.register_timeseries(device, &schema), common::E_OK);
+ ASSERT_EQ(writer.register_timeseries(valid_device, &valid_schema),
+ common::E_OK);
+ storage::TsRecord record(device, 0);
+ record.add_point(measurement, static_cast<int64_t>(42));
+ ASSERT_EQ(writer.write(record), common::E_OK);
+ storage::TsRecord valid_record(valid_device, 0);
+ valid_record.add_point(valid_measurement, static_cast<int64_t>(7));
+ ASSERT_EQ(writer.write(valid_record), common::E_OK);
+ ASSERT_EQ(writer.flush(), common::E_OK);
+ ASSERT_EQ(writer.close(), common::E_OK);
+ }
+
+ const std::string formats[] = {"csv", "ndjson", "table"};
+ for (const std::string& format : formats) {
+ for (const std::string& command : {"ls", "schema", "stats", "count"}) {
+ SCOPED_TRACE(command + " " + format);
+ std::ostringstream out, err;
+ ASSERT_EQ(
+ tsfile_cli::run_cli({command, "-f", format, path}, out, err), 0)
+ << err.str();
+ EXPECT_TRUE(tsfile_cli::is_valid_utf8(out.str()));
+ EXPECT_NE(out.str().find("root.device" + replacement),
+ std::string::npos);
+ EXPECT_NE(out.str().find(valid_device), std::string::npos);
+ }
+ for (const std::string& command : {"head", "cat"}) {
+ SCOPED_TRACE(command + " " + format);
+ std::ostringstream out, err;
+ ASSERT_EQ(tsfile_cli::run_cli({command, "-d", device, "-m",
+ measurement, "-f", format, path},
+ out, err),
+ 0)
+ << err.str();
+ EXPECT_TRUE(tsfile_cli::is_valid_utf8(out.str()));
+ EXPECT_NE(out.str().find("value" + replacement), std::string::npos);
+ EXPECT_NE(out.str().find("42"), std::string::npos);
+ }
+
+ const std::string output = tsfile_cli_test::unique_temp_path(
+ "tsfile_cli_utf8_export", "." + format);
+ std::ostringstream out, err;
+ ASSERT_EQ(tsfile_cli::run_cli({"export", "-d", device, "--type", format,
+ "-o", output, path},
+ out, err),
+ 0)
+ << err.str();
+ std::ifstream file(output.c_str(), std::ios::binary);
+ std::ostringstream content;
+ content << file.rdbuf();
+ EXPECT_TRUE(tsfile_cli::is_valid_utf8(content.str()));
+ EXPECT_NE(content.str().find("value" + replacement), std::string::npos);
+ file.close();
+ std::remove(output.c_str());
+ }
+
+ std::ostringstream sketch, sketch_err;
+ ASSERT_EQ(tsfile_cli::run_cli({"sketch", path}, sketch, sketch_err), 0)
+ << sketch_err.str();
+ EXPECT_TRUE(tsfile_cli::is_valid_utf8(sketch.str()));
+ EXPECT_NE(sketch.str().find("value" + replacement), std::string::npos);
+
+ const std::string dir =
+ tsfile_cli_test::unique_temp_path("tsfile_cli_utf8_manifest", "");
+ std::ostringstream out, err;
+ ASSERT_EQ(
+ tsfile_cli::run_cli({"export", "-d", device, "-d", valid_device,
+ "--type", "ndjson", "--output-dir", dir, path},
+ out, err),
+ 0)
+ << err.str();
+ std::ifstream manifest((dir + "/_manifest.json").c_str(), std::ios::binary);
+ std::ostringstream content;
+ content << manifest.rdbuf();
+ EXPECT_TRUE(tsfile_cli::is_valid_utf8(content.str()));
+ EXPECT_NE(content.str().find("root.device" + replacement),
+ std::string::npos);
+ EXPECT_NE(content.str().find(valid_device), std::string::npos);
+ manifest.close();
+ std::remove((dir + "/0001.ndjson").c_str());
+ std::remove((dir + "/0002.ndjson").c_str());
+ std::remove((dir + "/_manifest.json").c_str());
+#ifdef _WIN32
+ _rmdir(dir.c_str());
+#else
+ rmdir(dir.c_str());
+#endif
+ std::remove(path.c_str());
+}
+
+TEST(CliE2E, Utf8JsonKeyCollisionsFailWithoutPublishingOutput) {
+ storage::libtsfile_init();
+ const std::string path = tsfile_cli_test::unique_temp_path(
+ "tsfile_cli_utf8_collision", ".tsfile");
+ std::string device = "root.collision";
+ const std::vector<std::string> names = {"bad\xff", "bad\xfe",
+ "bad\xef\xbf\xbd"};
+ {
+ storage::WriteFile file;
+ int flags = O_WRONLY | O_CREAT | O_TRUNC;
+#ifdef _WIN32
+ flags |= O_BINARY;
+#endif
+ ASSERT_EQ(file.create(path, flags, 0666), common::E_OK);
+ storage::MeasurementSchema first(names[0], common::INT64, common::PLAIN,
+ common::UNCOMPRESSED);
+ storage::MeasurementSchema second(names[1], common::INT64,
+ common::PLAIN, common::UNCOMPRESSED);
+ storage::MeasurementSchema third(names[2], common::INT64, common::PLAIN,
+ common::UNCOMPRESSED);
+ storage::TsFileTreeWriter writer(&file);
+ ASSERT_EQ(writer.register_timeseries(device, &first), common::E_OK);
+ ASSERT_EQ(writer.register_timeseries(device, &second), common::E_OK);
+ ASSERT_EQ(writer.register_timeseries(device, &third), common::E_OK);
+ storage::TsRecord record(device, 0);
+ record.add_point(names[0], static_cast<int64_t>(42));
+ record.add_point(names[1], static_cast<int64_t>(7));
+ record.add_point(names[2], static_cast<int64_t>(9));
+ ASSERT_EQ(writer.write(record), common::E_OK);
+ ASSERT_EQ(writer.flush(), common::E_OK);
+ ASSERT_EQ(writer.close(), common::E_OK);
+ }
+
+ for (const std::string& command : {"head", "cat"}) {
+ for (size_t second : {size_t(1), size_t(2)}) {
+ for (bool empty : {false, true}) {
+ SCOPED_TRACE(command + " second=" + std::to_string(second) +
+ " empty=" + std::to_string(empty));
+ std::vector<std::string> args = {
+ command, "-d", device, "-m", names[0],
+ "-m", names[second], "-f", "ndjson", path};
+ if (empty) {
+ args.insert(args.end() - 1, {"--start", "0", "-n", "0"});
+ }
+ std::ostringstream out, err;
+ EXPECT_EQ(tsfile_cli::run_cli(args, out, err), 3) << err.str();
+ EXPECT_TRUE(out.str().empty());
+ EXPECT_NE(err.str().find("duplicate NDJSON column name"),
+ std::string::npos);
+ EXPECT_TRUE(tsfile_cli::is_valid_utf8(err.str()));
+ }
+ }
+ }
+
+ const std::string output = tsfile_cli_test::unique_temp_path(
+ "tsfile_cli_utf8_collision_export", ".ndjson");
+ for (bool existing : {false, true}) {
+ if (existing) {
+ std::ofstream target(output.c_str(), std::ios::binary);
+ target << "keep existing output\n";
+ }
+ std::vector<std::string> args = {"export", "-d", device, "--type",
+ "ndjson", "-o", output, path};
+ if (existing) {
+ args.insert(args.end() - 1, "--force");
+ }
+ std::ostringstream out, err;
+ EXPECT_EQ(tsfile_cli::run_cli(args, out, err), 3) << err.str();
+ EXPECT_TRUE(out.str().empty());
+ EXPECT_NE(err.str().find("duplicate NDJSON column name"),
+ std::string::npos);
+ std::ifstream target(output.c_str(), std::ios::binary);
+ if (existing) {
+ ASSERT_TRUE(target.is_open());
+ std::ostringstream content;
+ content << target.rdbuf();
+ EXPECT_EQ(content.str(), "keep existing output\n");
+ } else {
+ EXPECT_FALSE(target.is_open());
+ }
+ }
+ std::remove(output.c_str());
+ std::remove(path.c_str());
+}
+
+TEST(CliE2E, TableUtf8JsonKeyCollisionsFailInRowsAndStats) {
+ storage::libtsfile_init();
+ const std::string path = tsfile_cli_test::unique_temp_path(
+ "tsfile_cli_utf8_table_collision", ".tsfile");
+ std::string table = "collision";
+ const std::vector<std::string> names = {"bad\xff", "bad\xfe", "value"};
+ {
+ storage::WriteFile file;
+ int flags = O_WRONLY | O_CREAT | O_TRUNC;
+#ifdef _WIN32
+ flags |= O_BINARY;
+#endif
+ ASSERT_EQ(file.create(path, flags, 0666), common::E_OK);
+ storage::TableSchema schema(
+ table, {common::ColumnSchema(names[0], common::STRING,
+ common::UNCOMPRESSED, common::PLAIN,
+ common::ColumnCategory::TAG),
+ common::ColumnSchema(names[1], common::STRING,
+ common::UNCOMPRESSED, common::PLAIN,
+ common::ColumnCategory::TAG),
+ common::ColumnSchema(names[2], common::INT64,
+ common::UNCOMPRESSED, common::PLAIN,
+ common::ColumnCategory::FIELD)});
+ storage::TsFileTableWriter writer(&file, &schema);
+ storage::Tablet tablet(
+ table, names, {common::STRING, common::STRING, common::INT64},
+ {common::ColumnCategory::TAG, common::ColumnCategory::TAG,
+ common::ColumnCategory::FIELD},
+ 1);
+ tablet.add_timestamp(0, static_cast<int64_t>(0));
+ tablet.add_value(0, names[0], "first");
+ tablet.add_value(0, names[1], "second");
+ tablet.add_value(0, names[2], static_cast<int64_t>(42));
+ ASSERT_EQ(writer.write_table(tablet), common::E_OK);
+ ASSERT_EQ(writer.flush(), common::E_OK);
+ ASSERT_EQ(writer.close(), common::E_OK);
+ }
+ for (const std::string& command : {"head", "cat", "stats"}) {
+ SCOPED_TRACE(command);
+ std::ostringstream out, err;
+ EXPECT_EQ(tsfile_cli::run_cli(
+ {command, "-t", table, "-f", "ndjson", path}, out, err),
+ 3);
+ EXPECT_TRUE(out.str().empty());
+ EXPECT_NE(err.str().find("duplicate NDJSON column name"),
+ std::string::npos);
+ EXPECT_TRUE(tsfile_cli::is_valid_utf8(err.str()));
+ }
+ std::remove(path.c_str());
+}
diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc
index 55324ff3f..b58fafe3d 100644
--- a/cpp/test/tools/output_format_test.cc
+++ b/cpp/test/tools/output_format_test.cc
@@ -23,6 +23,7 @@
#include <sstream>
#include <streambuf>
+#include <utility>
#include <vector>
#include "common/db_common.h"
@@ -93,6 +94,56 @@ TEST(JsonEscapeTest, EscapesQuotesBackslashAndControls) {
EXPECT_EQ(tsfile_cli::json_escape("tab\there"), "tab\\there");
}
+TEST(Utf8OutputTest, PreservesValidSequencesAcrossFormats) {
+ const std::string text =
+ "ASCII\xc2\x80\xdf\xbf\xe0\xa0\x80\xe4\xb8\xad"
+ "\xed\x9f\xbf\xee\x80\x80\xef\xbb\xbf\xf0\x90\x80\x80"
+ "\xf0\x9f\x98\x80\xf4\x8f\xbf\xbf";
+ EXPECT_EQ(tsfile_cli::csv_escape(text), text);
+ EXPECT_EQ(tsfile_cli::json_escape(text), text);
+ EXPECT_EQ(tsfile_cli::table_escape(text), text);
+}
+
+TEST(Utf8OutputTest, ReplacesMaximalInvalidSubpartsAcrossFormats) {
+ const std::string replacement = "\xef\xbf\xbd";
+ const std::vector<std::pair<std::string, std::string>> cases = {
+ {"\xff", replacement},
+ {"\x80\xbf", replacement + replacement},
+ {"\xc0\xaf", replacement + replacement},
+ {"\xe0\x80\xbf", replacement + replacement + replacement},
+ {"\xed\xa0\x80", replacement + replacement + replacement},
+ {"\xf4\x91\x92\x93",
+ replacement + replacement + replacement + replacement},
+ {"\xc2", replacement},
+ {"\xe1\x80", replacement},
+ {"\xf0\x91\x92", replacement},
+ {"a\xc2"
+ "b",
+ "a" + replacement + "b"},
+ {"\xe1\x80"
+ "A",
+ replacement + "A"},
+ {"\xf1\xbf\xe4\xb8\xad", replacement + "\xe4\xb8\xad"},
+ {"\xe1\x80\xe2\xf0\x91\x92\xf1\xbf"
+ "A",
+ replacement + replacement + replacement + replacement + "A"},
+ };
+ for (const auto& test : cases) {
+ SCOPED_TRACE(::testing::PrintToString(test.first));
+ EXPECT_EQ(tsfile_cli::csv_escape(test.first), test.second);
+ EXPECT_EQ(tsfile_cli::json_escape(test.first), test.second);
+ EXPECT_EQ(tsfile_cli::table_escape(test.first), test.second);
+ }
+}
+
+TEST(Utf8OutputTest, PreservesDelimitersAfterAnInvalidSequence) {
+ const std::string text = "\xe1\x80\",\n";
+ const std::string replacement = "\xef\xbf\xbd";
+ EXPECT_EQ(tsfile_cli::csv_escape(text), "\"" + replacement + "\"\",\n\"");
+ EXPECT_EQ(tsfile_cli::json_escape(text), replacement + "\\\",\\n");
+ EXPECT_EQ(tsfile_cli::table_escape(text), replacement + "\",\\n");
+}
+
TEST(TableEscapeTest, EscapesBackslashAndNamedControls) {
EXPECT_EQ(tsfile_cli::table_escape("a\\b"), "a\\\\b");
EXPECT_EQ(tsfile_cli::table_escape("line\nbreak"), "line\\nbreak");
@@ -117,8 +168,7 @@ TEST(TableEscapeTest, ControlByteBoundaries) {
// Space (0x20) is printable: it must survive verbatim, never be escaped.
EXPECT_EQ(tsfile_cli::table_escape("a b"), "a b");
- // Bytes >= 0x80 are UTF-8 lead/continuation bytes; escaping them would
- // corrupt multi-byte sequences, so they must pass through unchanged.
+ // Well-formed UTF-8 must pass through unchanged.
EXPECT_EQ(tsfile_cli::table_escape("\xe4\xb8\xad"), "\xe4\xb8\xad");
}
@@ -264,3 +314,59 @@ TEST(RowWriterTest, ReportsFlushFailure) {
ASSERT_TRUE(writer.write({"value"}, {false}));
EXPECT_FALSE(writer.finish());
}
+
+TEST(RowWriterTest, ReplacesInvalidUtf8InHeadersAndValues) {
+ const std::string header = "name\xff";
+ const std::string value = "value\xe1\x80";
+ const std::string replacement = "\xef\xbf\xbd";
+ const OutputFormat formats[] = {OutputFormat::kCsv, OutputFormat::kJson,
+ OutputFormat::kTable};
+ for (OutputFormat format : formats) {
+ std::ostringstream out;
+ RowWriter writer(out, format, {header}, {common::STRING}, false);
+ ASSERT_TRUE(writer.write({value}, {false}));
+ ASSERT_TRUE(writer.finish());
+ const std::string expected =
+ format == OutputFormat::kJson
+ ? "{\"name" + replacement + "\":\"value" + replacement + "\"}\n"
+ : "name" + replacement + "\nvalue" + replacement + "\n";
+ EXPECT_EQ(out.str(), expected);
+ }
+}
+
+TEST(RowWriterTest, PreservesNonUtf8BlobBytesAsHex) {
+ const OutputFormat formats[] = {OutputFormat::kCsv, OutputFormat::kJson,
+ OutputFormat::kTable};
+ for (OutputFormat format : formats) {
+ std::ostringstream out;
+ RowWriter writer(out, format, {"payload"}, {common::BLOB}, false);
+ ASSERT_TRUE(writer.write({std::string("\xff\0\x80", 3)}, {false}));
+ ASSERT_TRUE(writer.finish());
+ EXPECT_NE(out.str().find("0xff0080"), std::string::npos);
+ }
+}
+
+TEST(RowWriterTest, RejectsJsonKeyCollisionsAfterUtf8Replacement) {
+ const std::vector<std::vector<std::string>> headers = {
+ {"bad\xff", "bad\xfe"},
+ {"bad\xff", "bad\xef\xbf\xbd"},
+ };
+ for (const auto& header : headers) {
+ for (bool no_header : {false, true}) {
+ std::ostringstream out;
+ RowWriter writer(out, OutputFormat::kJson, header,
+ {common::INT64, common::INT64}, no_header);
+ EXPECT_FALSE(writer.write({"42", "7"}, {false, false}));
+ EXPECT_FALSE(writer.finish());
+ EXPECT_TRUE(out.str().empty());
+ }
+ }
+}
+
+TEST(RowWriterTest, RejectsJsonKeyCollisionsWithoutRows) {
+ std::ostringstream out;
+ RowWriter writer(out, OutputFormat::kJson, {"bad\xff", "bad\xfe"},
+ {common::STRING, common::STRING}, false);
+ EXPECT_FALSE(writer.finish());
+ EXPECT_TRUE(out.str().empty());
+}
diff --git a/cpp/tools/README.md b/cpp/tools/README.md
index 37f704896..874f52bb0 100644
--- a/cpp/tools/README.md
+++ b/cpp/tools/README.md
@@ -121,6 +121,12 @@ values. The `table` format uses a temporary spool to align columns with bounded
memory; prefer `csv`/`ndjson` when temporary disk use is undesirable. `sketch`
does not accept `--format`.
+Text output preserves well-formed UTF-8 and replaces malformed UTF-8 in names
+and text with U+FFFD (`�`). This also applies to sketch output and export
+manifests. BLOB values remain hexadecimal strings.
+NDJSON output fails before writing rows if replacement would produce duplicate
+column names.
+
```bash
BIN=cpp/build/Debug/bin/tsfile-cli
$BIN ls -f csv data.tsfile # list tables / devices
diff --git a/cpp/tools/commands/cmd_export.cc b/cpp/tools/commands/cmd_export.cc
index ecca955e1..9a7a1c7ee 100644
--- a/cpp/tools/commands/cmd_export.cc
+++ b/cpp/tools/commands/cmd_export.cc
@@ -37,6 +37,7 @@
#include "common/device_id.h"
#include "common/schema.h"
#include "format/atomic_output.h"
+#include "format/output_format.h"
#include "reader/tsfile_reader.h"
namespace tsfile_cli {
@@ -143,32 +144,6 @@ std::string numbered_file_name(size_t index, ParsedArgs::Format fmt) {
return std::string(buf) + extension_for_format(fmt);
}
-std::string json_escape(const std::string& s) {
- std::ostringstream out;
- for (char c : s) {
- switch (c) {
- case '\\':
- out << "\\\\";
- break;
- case '"':
- out << "\\\"";
- break;
- case '\n':
- out << "\\n";
- break;
- case '\r':
- out << "\\r";
- break;
- case '\t':
- out << "\\t";
- break;
- default:
- out << c;
- }
- }
- return out.str();
-}
-
struct ManifestEntry {
std::string file;
std::string model;
diff --git a/cpp/tools/commands/cmd_sketch.cc b/cpp/tools/commands/cmd_sketch.cc
index eb762048f..eff543fb2 100644
--- a/cpp/tools/commands/cmd_sketch.cc
+++ b/cpp/tools/commands/cmd_sketch.cc
@@ -128,8 +128,7 @@ int write_atomic_text(const std::string& path, const std::string& content,
return code;
}
{
- std::ofstream output(tmp.c_str(),
- std::ios::binary | std::ios::trunc);
+ std::ofstream output(tmp.c_str(), std::ios::binary | std::ios::trunc);
if (!output.is_open()) {
err << "Error: cannot create output target '" << path << "'\n";
remove_atomic_temp(tmp, err);
@@ -460,8 +459,7 @@ class SketchPrinter {
to_string_u32(layout_.footer_.bloom_filter_hash_count));
std::ostringstream bloom;
bloom << "[Bloom Filter] , filterCapacity="
- << layout_.footer_.bloom_filter_size
- << ", hashFunctionSize="
+ << layout_.footer_.bloom_filter_size << ", hashFunctionSize="
<< layout_.footer_.bloom_filter_hash_count;
print_line(out, layout_.footer_.bloom_filter_offset, bloom.str());
}
@@ -587,21 +585,16 @@ int cmd_sketch(const ParsedArgs& args, std::ostream& out, std::ostream& err) {
SketchPrinter printer;
std::ostringstream content;
int code = printer.run(args, content, err);
- if (code != kExitOk) {
- if (args.output.empty()) {
- out << content.str();
- out.flush();
- return out.good() ? code : kExitRuntime;
- }
+ if (code != kExitOk && !args.output.empty()) {
return code;
}
+ const std::string text = replace_invalid_utf8(content.str());
if (!args.output.empty()) {
- return write_atomic_text(args.output, content.str(), args.file,
- args.force, err);
+ return write_atomic_text(args.output, text, args.file, args.force, err);
}
- out << content.str();
+ out << text;
out.flush();
- return out.good() ? kExitOk : kExitRuntime;
+ return out.good() ? code : kExitRuntime;
}
} // namespace tsfile_cli
diff --git a/cpp/tools/commands/cmd_stats.cc b/cpp/tools/commands/cmd_stats.cc
index 01bd5ba57..df5371f1b 100644
--- a/cpp/tools/commands/cmd_stats.cc
+++ b/cpp/tools/commands/cmd_stats.cc
@@ -500,6 +500,10 @@ int cmd_table_stats(const ParsedArgs& args, storage::TsFileReader& reader,
types.push_back(common::STRING);
}
RowWriter w(out, fmt, headers, types, args.no_header);
+ if (!w.error().empty()) {
+ err << "Error: " << w.error() << "\n";
+ return kExitRuntime;
+ }
for (const TableStatsSummary& summary : summaries) {
std::map<std::string, size_t> local_tag_positions;
for (size_t i = 0; i < summary.tag_indexes.size(); ++i) {
diff --git a/cpp/tools/commands/row_query.cc b/cpp/tools/commands/row_query.cc
index 828bb6c55..f6150600f 100644
--- a/cpp/tools/commands/row_query.cc
+++ b/cpp/tools/commands/row_query.cc
@@ -361,11 +361,17 @@ int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader,
// that callers could mistake for a complete result. The final write is
// still checked separately so stdout errors remain runtime failures.
std::ostringstream staged;
- int wret = push_down ? emit_result_set(rs, fmt, args.no_header, staged, 0,
- -1, emitted_rows)
- : emit_result_set(rs, fmt, args.no_header, staged,
- offset, limit, emitted_rows);
+ std::string output_error;
+ int wret = push_down
+ ? emit_result_set(rs, fmt, args.no_header, staged, 0, -1,
+ emitted_rows, &output_error)
+ : emit_result_set(rs, fmt, args.no_header, staged, offset,
+ limit, emitted_rows, &output_error);
reader.destroy_query_data_set(rs);
+ if (!output_error.empty()) {
+ err << "Error: " << output_error << "\n";
+ return kExitRuntime;
+ }
if (wret == common::E_OK) {
const std::string bytes = staged.str();
out.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc
index 63aa39ad6..b6ba39c1b 100644
--- a/cpp/tools/format/output_format.cc
+++ b/cpp/tools/format/output_format.cc
@@ -21,6 +21,7 @@
#include <algorithm>
#include <cstdio>
+#include <set>
#include <utility>
#include "common/csv_utils.h"
@@ -192,14 +193,72 @@ const char* compression_name(common::CompressionType c) {
}
}
+namespace {
+
+// UTF-8 encoding of the replacement character U+FFFD.
+constexpr unsigned char kUtf8Replacement[] = {0xef, 0xbf, 0xbd};
+
+template <typename Emit>
+void for_each_utf8_byte(const std::string& s, Emit emit) {
+ size_t i = 0;
+ while (i < s.size()) {
+ const unsigned char lead = static_cast<unsigned char>(s[i]);
+ size_t length = 0;
+ if (lead <= 0x7f) {
+ length = 1;
+ } else if (lead >= 0xc2 && lead <= 0xdf) {
+ length = 2;
+ } else if (lead >= 0xe0 && lead <= 0xef) {
+ length = 3;
+ } else if (lead >= 0xf0 && lead <= 0xf4) {
+ length = 4;
+ }
+ size_t consumed = 1;
+ while (consumed < length && consumed < s.size() - i) {
+ const unsigned char next =
+ static_cast<unsigned char>(s[i + consumed]);
+ if (next < 0x80 || next > 0xbf ||
+ (consumed == 1 && ((lead == 0xe0 && next < 0xa0) ||
+ (lead == 0xed && next > 0x9f) ||
+ (lead == 0xf0 && next < 0x90) ||
+ (lead == 0xf4 && next > 0x8f)))) {
+ break;
+ }
+ ++consumed;
+ }
+ if (consumed == length) {
+ for (size_t j = 0; j < consumed; ++j) {
+ emit(static_cast<unsigned char>(s[i + j]));
+ }
+ } else {
+ for (unsigned char c : kUtf8Replacement) {
+ emit(c);
+ }
+ }
+ // Consume only a valid prefix of this character, leaving subsequent
+ // ASCII or valid UTF-8 for the next iteration, even after truncation.
+ i += consumed;
+ }
+}
+
+} // namespace
+
+std::string replace_invalid_utf8(const std::string& s) {
+ std::string out;
+ out.reserve(s.size());
+ for_each_utf8_byte(
+ s, [&out](unsigned char c) { out += static_cast<char>(c); });
+ return out;
+}
+
std::string csv_escape(const std::string& field) {
- return common::csv_escape(field, ',');
+ return common::csv_escape(replace_invalid_utf8(field), ',');
}
std::string json_escape(const std::string& s) {
std::string out;
out.reserve(s.size() + 2);
- for (unsigned char c : s) {
+ for_each_utf8_byte(s, [&out](unsigned char c) {
switch (c) {
case '"':
out += "\\\"";
@@ -231,7 +290,7 @@ std::string json_escape(const std::string& s) {
out += static_cast<char>(c);
}
}
- }
+ });
return out;
}
@@ -243,7 +302,7 @@ std::string json_escape(const std::string& s) {
std::string table_escape(const std::string& s) {
std::string out;
out.reserve(s.size() + 2);
- for (unsigned char c : s) {
+ for_each_utf8_byte(s, [&out](unsigned char c) {
switch (c) {
case '\\':
out += "\\\\";
@@ -260,8 +319,8 @@ std::string table_escape(const std::string& s) {
default:
// C0 controls (0x00-0x1f) plus DEL (0x7f) are the complete
// set of non-printable control bytes per C iscntrl(). Bytes
- // >= 0x80 are UTF-8 continuation/lead bytes and must pass
- // through untouched so multi-byte sequences survive.
+ // >= 0x80 now belong to well-formed UTF-8 and pass through
+ // untouched so multi-byte sequences survive.
if (c < 0x20 || c == 0x7f) {
char buf[8];
std::snprintf(buf, sizeof(buf), "\\u%04x", c);
@@ -270,7 +329,7 @@ std::string table_escape(const std::string& s) {
out += static_cast<char>(c);
}
}
- }
+ });
return out;
}
@@ -308,7 +367,21 @@ RowWriter::RowWriter(std::ostream& out, OutputFormat fmt,
types_(std::move(types)),
no_header_(no_header),
table_widths_(header_.size(), 0) {
- if (!no_header_) {
+ if (fmt_ == OutputFormat::kJson) {
+ std::set<std::string> keys;
+ json_keys_.reserve(header_.size());
+ for (const std::string& name : header_) {
+ std::string key = json_escape(name);
+ if (!keys.insert(key).second) {
+ error_ =
+ "duplicate NDJSON column name after UTF-8 replacement: \"" +
+ key + "\"";
+ return;
+ }
+ json_keys_.push_back(std::move(key));
+ }
+ }
+ if (fmt_ == OutputFormat::kTable && !no_header_) {
for (size_t i = 0; i < header_.size(); ++i) {
table_widths_[i] = table_escape(header_[i]).size();
}
@@ -372,7 +445,7 @@ bool RowWriter::write(const std::vector<std::string>& cells,
bool RowWriter::write(const std::vector<std::string>& cells,
const std::vector<bool>& is_null,
const std::vector<common::TSDataType>& row_types) {
- if (!out_.good()) {
+ if (!error_.empty() || !out_.good()) {
return false;
}
if (fmt_ == OutputFormat::kTable) {
@@ -410,7 +483,7 @@ bool RowWriter::write(const std::vector<std::string>& cells,
if (i) {
out_ << ",";
}
- out_ << "\"" << json_escape(header_[i]) << "\":";
+ out_ << "\"" << json_keys_[i] << "\":";
const common::TSDataType type =
i < row_types.size() ? row_types[i] : common::STRING;
if (i < is_null.size() && is_null[i]) {
@@ -463,7 +536,7 @@ bool RowWriter::write(const std::vector<std::string>& cells,
}
bool RowWriter::finish() {
- if (!out_.good()) {
+ if (!error_.empty() || !out_.good()) {
return false;
}
if (fmt_ != OutputFormat::kTable) {
diff --git a/cpp/tools/format/output_format.h b/cpp/tools/format/output_format.h
index 1a9ed52e9..02d005b41 100644
--- a/cpp/tools/format/output_format.h
+++ b/cpp/tools/format/output_format.h
@@ -42,6 +42,8 @@ const char* tsdatatype_name(common::TSDataType t);
const char* tsencoding_name(common::TSEncoding e);
const char* compression_name(common::CompressionType c);
+// Replace each maximal ill-formed UTF-8 subpart with U+FFFD (Unicode 3.9.6).
+std::string replace_invalid_utf8(const std::string& s);
std::string csv_escape(const std::string& field);
std::string json_escape(const std::string& s);
std::string table_escape(const std::string& s);
@@ -60,6 +62,9 @@ class RowWriter {
const std::vector<common::TSDataType>& row_types);
bool finish();
+ // Header validation failures are reported before any output is written.
+ const std::string& error() const { return error_; }
+
private:
bool ensure_header();
bool emits_json_bare(common::TSDataType type) const;
@@ -69,6 +74,8 @@ class RowWriter {
std::ostream& out_;
OutputFormat fmt_;
std::vector<std::string> header_;
+ std::vector<std::string> json_keys_;
+ std::string error_;
std::vector<common::TSDataType> types_;
bool no_header_;
bool header_done_ = false;
diff --git a/cpp/tools/format/result_set_format.cc b/cpp/tools/format/result_set_format.cc
index ea0b350b4..a67e39c8c 100644
--- a/cpp/tools/format/result_set_format.cc
+++ b/cpp/tools/format/result_set_format.cc
@@ -74,7 +74,10 @@ std::string cell_to_string(storage::ResultSet* rs, uint32_t i,
int emit_result_set(storage::ResultSet* rs, OutputFormat fmt, bool no_header,
std::ostream& out, long long offset, long long limit,
- long long* emitted_rows) {
+ long long* emitted_rows, std::string* output_error) {
+ if (output_error != nullptr) {
+ output_error->clear();
+ }
auto meta = rs->get_metadata();
const uint32_t ncol = meta->get_column_count();
std::vector<std::string> header;
@@ -87,6 +90,12 @@ int emit_result_set(storage::ResultSet* rs, OutputFormat fmt, bool no_header,
}
RowWriter writer(out, fmt, header, types, no_header);
+ if (!writer.error().empty()) {
+ if (output_error != nullptr) {
+ *output_error = writer.error();
+ }
+ return common::E_INVALID_ARG;
+ }
bool has_next = false;
int code = common::E_OK;
long long skipped = 0;
diff --git a/cpp/tools/format/result_set_format.h b/cpp/tools/format/result_set_format.h
index 40814a47b..84ea8f0e9 100644
--- a/cpp/tools/format/result_set_format.h
+++ b/cpp/tools/format/result_set_format.h
@@ -32,9 +32,11 @@ namespace tsfile_cli {
std::string cell_to_string(storage::ResultSet* rs, uint32_t col_index,
common::TSDataType type);
+// output_error receives a diagnostic when output column names are invalid.
int emit_result_set(storage::ResultSet* rs, OutputFormat fmt, bool no_header,
std::ostream& out, long long offset = 0,
- long long limit = -1, long long* emitted_rows = nullptr);
+ long long limit = -1, long long* emitted_rows = nullptr,
+ std::string* output_error = nullptr);
int emit_result_set_sampled(storage::ResultSet* rs, OutputFormat fmt,
bool no_header, std::ostream& out, long long limit,